diff --git a/Library/DiscUtils.Iscsi/DiscUtils.Iscsi.csproj b/Library/DiscUtils.Iscsi/DiscUtils.Iscsi.csproj index 484acbe48..b1344c2da 100644 --- a/Library/DiscUtils.Iscsi/DiscUtils.Iscsi.csproj +++ b/Library/DiscUtils.Iscsi/DiscUtils.Iscsi.csproj @@ -9,4 +9,8 @@ + + + + diff --git a/Library/DiscUtils.Iscsi/Disk.cs b/Library/DiscUtils.Iscsi/Disk.cs index e2cfebc9f..45b8a7be8 100644 --- a/Library/DiscUtils.Iscsi/Disk.cs +++ b/Library/DiscUtils.Iscsi/Disk.cs @@ -164,4 +164,29 @@ public override VirtualDisk CreateDifferencingDisk(string path, bool useAsync = { throw new NotSupportedException("Differencing disks not supported for iSCSI disks"); } -} \ No newline at end of file + + /// + /// Disposes of this instance. Like an operating system that unmounts a disk, it first asks the target to put + /// the writes it acknowledged on stable storage. + /// + /// true if running inside Dispose(). + /// + /// A failure of that last flush is not reported here (the session may already be gone). Call + /// Content.Flush() before disposing to find out whether the target stored the writes. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + _stream?.Flush(flushToDisk: true); + } + catch (Exception e) when (e is IOException or ObjectDisposedException) + { + } + } + + base.Dispose(disposing); + } +} diff --git a/Library/DiscUtils.Iscsi/DiskStream.cs b/Library/DiscUtils.Iscsi/DiskStream.cs index a1a822f87..9a505ace4 100644 --- a/Library/DiscUtils.Iscsi/DiskStream.cs +++ b/Library/DiscUtils.Iscsi/DiskStream.cs @@ -71,7 +71,24 @@ public override long Position set => _position = value; } - public override void Flush() { } + /// + /// Requests that the target synchronize its write cache with stable storage (SCSI SYNCHRONIZE CACHE). + /// This covers the entire LUN, including writes made through other streams or before this stream was opened. + /// A failed or unsupported synchronization is reported to the caller. + /// + public override void Flush() => Flush(flushToDisk: true); + + /// + /// Requests a target-wide cache synchronization. iSCSI has no separate buffered data in this stream to flush, + /// so both overloads preserve the established behavior and issue SYNCHRONIZE CACHE. + /// In particular, set to true is an unconditional durability barrier, + /// even if this stream has not written any data itself. + /// + public override void Flush(bool flushToDisk) => _session.SynchronizeCache(_lun); + + /// Asynchronously synchronizes the whole LUN, even if this stream has not written anything. + public override async Task FlushAsync(CancellationToken cancellationToken) + => await _session.SynchronizeCacheAsync(_lun, cancellationToken).ConfigureAwait(false); public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); @@ -242,4 +259,4 @@ public override async ValueTask WriteAsync(ReadOnlyMemory buffer, Cancella } } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Iscsi/README.md b/Library/DiscUtils.Iscsi/README.md index 51ec60960..95815b634 100644 --- a/Library/DiscUtils.Iscsi/README.md +++ b/Library/DiscUtils.Iscsi/README.md @@ -22,6 +22,12 @@ dotnet add package LTRData.DiscUtils.Iscsi Initiator and Session provide explicit connections and LUN access. The package also supplies an iscsi transport for generic disk opening. This is a client library, not an iSCSI server or an operating-system initiator driver. Registration is AOT-compatible; protocol-key reflection elsewhere in iSCSI is outside that guarantee. +### Write-cache synchronization + +`Disk.Content.Flush()` and `Disk.Content.Flush(true)` issue SCSI `SYNCHRONIZE CACHE (10)` for the entire LUN, with `IMMED` clear. The request is sent even when the current stream has not written anything: a stream-local dirty flag cannot establish whether the target has outstanding cached writes from an earlier session or another stream. `FlushAsync()` requests the same synchronization. + +An unsupported command (including SCSI `ILLEGAL REQUEST` / ASC `0x20`) or other synchronization failure is reported to the caller. An unsupported command does **not** establish that the target has no volatile write cache. The `Disk` disposal flush remains best-effort and suppresses errors; applications that need a durability barrier must call and check `Content.Flush(true)` explicitly before disposal. Even a successful command depends on the target and backing storage honoring SCSI cache synchronization. + ## Registration For automatic discovery, register the providers implemented by this package: diff --git a/Library/DiscUtils.Iscsi/ScsiSynchronizeCacheCommand.cs b/Library/DiscUtils.Iscsi/ScsiSynchronizeCacheCommand.cs new file mode 100644 index 000000000..6ff7d9156 --- /dev/null +++ b/Library/DiscUtils.Iscsi/ScsiSynchronizeCacheCommand.cs @@ -0,0 +1,31 @@ +using System; + +namespace DiscUtils.Iscsi; + +/// +/// SYNCHRONIZE CACHE (10): asks the target to put every write it has acknowledged on stable storage. +/// +/// +/// LBA 0 with 0 blocks covers the whole medium, also beyond what a 32-bit LBA can address. IMMED is clear, so the +/// target only answers once the data is stored. +/// +internal class ScsiSynchronizeCache10Command : ScsiCommand +{ + public ScsiSynchronizeCache10Command(ulong targetLun) + : base(targetLun) { } + + public override int Size => 10; + + public override TaskAttributes TaskAttributes => TaskAttributes.Simple; + + public override int ReadFrom(ReadOnlySpan buffer) + { + throw new NotImplementedException(); + } + + public override void WriteTo(Span buffer) + { + buffer[..10].Clear(); + buffer[0] = (byte)ScsiOpCode.SynchronizeCache10; + } +} diff --git a/Library/DiscUtils.Iscsi/Session.cs b/Library/DiscUtils.Iscsi/Session.cs index 4dd74e4fd..3439e7e85 100644 --- a/Library/DiscUtils.Iscsi/Session.cs +++ b/Library/DiscUtils.Iscsi/Session.cs @@ -387,6 +387,25 @@ public int RawCommand(long lun, ReadOnlyMemory command, ReadOnlySpan return Send(cmd, outBuffer, inBuffer); } + /// + /// Asks the target to put every write to a LUN that it has acknowledged on stable storage (SCSI SYNCHRONIZE CACHE). + /// + /// The LUN to flush. + internal void SynchronizeCache(long lun) + { + Send(new ScsiSynchronizeCache10Command((ulong)lun), default, default); + } + + /// + /// Asks the target to put every write to a LUN that it has acknowledged on stable storage (SCSI SYNCHRONIZE CACHE). + /// + /// The LUN to flush. + /// + internal async ValueTask SynchronizeCacheAsync(long lun, CancellationToken cancellationToken) + { + await SendAsync(new ScsiSynchronizeCache10Command((ulong)lun), default, default, cancellationToken).ConfigureAwait(false); + } + internal uint NextCommandSequenceNumber() { return ++CommandSequenceNumber; diff --git a/Library/DiscUtils.Ntfs/Directory.cs b/Library/DiscUtils.Ntfs/Directory.cs index f14d871e4..f517971b8 100644 --- a/Library/DiscUtils.Ntfs/Directory.cs +++ b/Library/DiscUtils.Ntfs/Directory.cs @@ -109,7 +109,7 @@ public override string ToString() return dir; } - internal DirectoryEntry? GetEntryByName(string name) + internal DirectoryEntry? GetEntryByName(string name, bool requireUnique = false) { var searchName = name; @@ -119,13 +119,22 @@ public override string ToString() searchName = name[..streamSepPos]; } - var entry = Index.FindFirst(new FileNameQuery(searchName, _context.UpperCase)); - if (entry.Key != null) + using var matches = Index.FindAll(new FileNameQuery(searchName, _context.UpperCase), validate: requireUnique).GetEnumerator(); + if (!matches.MoveNext()) { - return new DirectoryEntry(this, entry.Value, entry.Key); + return null; } - return null; + var entry = matches.Current; + while (requireUnique && matches.MoveNext()) + { + if (matches.Current.Value != entry.Value) + { + throw new InvalidDataException($"The directory has ambiguous entries for {name}"); + } + } + + return new DirectoryEntry(this, entry.Value, entry.Key); } internal DirectoryEntry AddEntry(File file, string name, FileNameNamespace nameNamespace) diff --git a/Library/DiscUtils.Ntfs/FileSystemFactory.cs b/Library/DiscUtils.Ntfs/FileSystemFactory.cs index c1649f654..e4a53b7da 100644 --- a/Library/DiscUtils.Ntfs/FileSystemFactory.cs +++ b/Library/DiscUtils.Ntfs/FileSystemFactory.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. // +using System; using System.Collections.Generic; using System.IO; using DiscUtils.Vfs; @@ -42,6 +43,19 @@ public override IEnumerable Detect(Stream stream, VolumeInfo vol private NtfsFileSystem Open(Stream stream, VolumeInfo volumeInfo, FileSystemParameters parameters) { - return new NtfsFileSystem(stream); + const string key = "NtfsJournalMode"; + if (parameters == null || !parameters.Parameters.ContainsKey(key)) + { + return new NtfsFileSystem(stream); + } + + var text = parameters.Parameters[key]?.Trim(); + if (!Enum.TryParse(text, ignoreCase: true, out var mode) || + !string.Equals(text, Enum.GetName(typeof(NtfsJournalMode), mode), StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException($"{key} must be Replay, Refuse or Ignore", nameof(parameters)); + } + + return new NtfsFileSystem(stream, mode); } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Ntfs/Index.cs b/Library/DiscUtils.Ntfs/Index.cs index d482c88bf..5b2bf2934 100644 --- a/Library/DiscUtils.Ntfs/Index.cs +++ b/Library/DiscUtils.Ntfs/Index.cs @@ -199,9 +199,16 @@ public static void Create(AttributeType attrType, AttributeCollationRule collati idx.WriteRootNodeToDisk(); } - public IEnumerable> FindAll(IComparable query) + public IEnumerable> FindAll(IComparable query, bool validate = false) { - foreach (var entry in FindAllIn(query, _rootNode)) + if (validate && (!IsFileIndex || _root.AttributeType != (uint)AttributeType.FileName || + _root.CollationRule != AttributeCollationRule.Filename || _root.IndexAllocationSize != _bpb.IndexBufferSize || + _root.RawClustersPerIndexRecord == 0)) + { + throw new InvalidDataException("The directory index root has an invalid format"); + } + + foreach (var entry in FindAllIn(query, _rootNode, validate, depth: 0)) { yield return new KeyValuePair(entry.KeyBuffer, entry.DataBuffer); } @@ -414,8 +421,13 @@ protected IEnumerable Enumerate(IndexNode node) } } - private IEnumerable FindAllIn(IComparable query, IndexNode node) + private IEnumerable FindAllIn(IComparable query, IndexNode node, bool validate, int depth) { + if (validate) + { + ValidateLookupNode(node, depth); + } + foreach (var focus in node.Entries) { var searchChildren = true; @@ -441,8 +453,13 @@ private IEnumerable FindAllIn(IComparable query, IndexNode n if (searchChildren && (focus.Flags & IndexEntryFlags.Node) != 0) { + if (validate) + { + ValidateLookupChild(focus.ChildrenVirtualCluster); + } + var block = GetSubBlock(focus); - foreach (var entry in FindAllIn(query, block.Node)) + foreach (var entry in FindAllIn(query, block.Node, validate, depth + 1)) { yield return entry; } @@ -460,6 +477,58 @@ private IEnumerable FindAllIn(IComparable query, IndexNode n } } + private void ValidateLookupNode(IndexNode node, int depth) + { + var header = node.Header; + if (depth >= 64 || header.OffsetToFirstEntry < IndexHeader.Size || + header.OffsetToFirstEntry + (long)node.CalcEntriesSize() != header.TotalSizeOfEntries || + header.TotalSizeOfEntries > header.AllocatedSizeOfEntries) + { + throw new InvalidDataException("The directory index has invalid node bounds or depth"); + } + + byte[] previousKey = null; + foreach (var entry in node.Entries) + { + if ((entry.Flags & ~(IndexEntryFlags.Node | IndexEntryFlags.End)) != 0) + { + throw new InvalidDataException("The directory index has an invalid entry flag"); + } + + if ((entry.Flags & IndexEntryFlags.End) != 0) + { + return; + } + + var key = entry.KeyBuffer; + if (key.Length < 0x42 || key.Length != 0x42 + 2 * key[0x40] || + (previousKey != null && _comparer.Compare(previousKey, key) > 0)) + { + throw new InvalidDataException("The directory index has an invalid or out-of-order filename"); + } + + previousKey = key; + } + + throw new InvalidDataException("The directory index node has no end entry"); + } + + private void ValidateLookupChild(long vcn) + { + if (vcn < 0 || vcn > long.MaxValue / Math.Max(_bpb.BytesPerCluster, (long)IndexBufferSize) || + AllocationStream == null || _indexBitmap == null) + { + throw new InvalidDataException("The directory index has an invalid child reference"); + } + + var position = IndexBlockVcnToPosition(vcn); + if (position > AllocationStream.Length - IndexBufferSize || position % IndexBufferSize != 0 || + !_indexBitmap.IsPresent(position / IndexBufferSize)) + { + throw new InvalidDataException("The directory index child is outside the allocated index blocks"); + } + } + private void WriteRootNodeToDisk() { _rootNode.Header.AllocatedSizeOfEntries = (uint)_rootNode.CalcSize(); diff --git a/Library/DiscUtils.Ntfs/IndexView.cs b/Library/DiscUtils.Ntfs/IndexView.cs index 6961f88de..86dc7b645 100644 --- a/Library/DiscUtils.Ntfs/IndexView.cs +++ b/Library/DiscUtils.Ntfs/IndexView.cs @@ -57,9 +57,9 @@ public D this[K key] set => _index[Unconvert(key)] = Unconvert(value); } - public IEnumerable> FindAll(IComparable query) + public IEnumerable> FindAll(IComparable query, bool validate = false) { - foreach (var entry in _index.FindAll(query)) + foreach (var entry in _index.FindAll(query, validate)) { yield return new KeyValuePair(Convert(entry.Key), Convert(entry.Value)); } diff --git a/Library/DiscUtils.Ntfs/Journal/JournalUtilities.cs b/Library/DiscUtils.Ntfs/Journal/JournalUtilities.cs new file mode 100644 index 000000000..4d785516c --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/JournalUtilities.cs @@ -0,0 +1,252 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.IO; +using DiscUtils.Streams; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// The journal could not be read or replayed. The volume is left as it was. +/// +internal sealed class JournalException : IOException +{ + public JournalException(string message) + : base(message) { } +} + +internal static class JournalUtilities +{ + public static readonly uint SignatureFile = Signature("FILE"); + public static readonly uint SignatureIndx = Signature("INDX"); + public static readonly uint SignatureRstr = Signature("RSTR"); + public static readonly uint SignatureRcrd = Signature("RCRD"); + public static readonly uint SignatureChkd = Signature("CHKD"); + public static readonly uint SignatureBaad = Signature("BAAD"); + public static readonly uint SignatureHole = Signature("HOLE"); + + private static uint Signature(string text) => + (uint)(text[0] | (text[1] << 8) | (text[2] << 16) | (text[3] << 24)); + + public static ushort U16(byte[] b, int offset) => EndianUtilities.ToUInt16LittleEndian(b, offset); + + public static uint U32(byte[] b, int offset) => EndianUtilities.ToUInt32LittleEndian(b, offset); + + public static ulong U64(byte[] b, int offset) => EndianUtilities.ToUInt64LittleEndian(b, offset); + + public static void Put16(byte[] b, int offset, ushort value) => EndianUtilities.WriteBytesLittleEndian(value, b, offset); + + public static void Put32(byte[] b, int offset, uint value) => EndianUtilities.WriteBytesLittleEndian(value, b, offset); + + public static void Put64(byte[] b, int offset, ulong value) => EndianUtilities.WriteBytesLittleEndian(value, b, offset); + + public static int Align8(int value) => (value + 7) & ~7; + + /// + /// Removes the update sequence array protection of a multi-sector structure (MFT record, index buffer, + /// log page) in place: the last two bytes of every stride go back to the values saved in the array. + /// + /// false if the header is implausible or a stride does not end with the sequence number + /// (a torn write); the buffer is then left unchanged. + public static bool RemoveFixups(byte[] buffer, int offset, int length) + { + if (!TryGetFixupLayout(buffer, offset, length, out var usaOffset, out var usaCount, out var stride)) + { + return false; + } + + var usn = U16(buffer, offset + usaOffset); + for (var i = 1; i < usaCount; i++) + { + if (U16(buffer, offset + (i * stride) - 2) != usn) + { + return false; + } + } + + for (var i = 1; i < usaCount; i++) + { + buffer[offset + (i * stride) - 2] = buffer[offset + usaOffset + (2 * i)]; + buffer[offset + (i * stride) - 1] = buffer[offset + usaOffset + (2 * i) + 1]; + } + + return true; + } + + /// + /// Protects a multi-sector structure before it is written: increments the update sequence number and puts it + /// at the end of every stride, saving the original bytes in the array. + /// + public static void ApplyFixups(byte[] buffer, int offset, int length, byte[] replacedBytes = null) + { + if (!TryGetFixupLayout(buffer, offset, length, out var usaOffset, out var usaCount, out var stride)) + { + throw new JournalException("A changed record has no valid update sequence array"); + } + + var usn = NextUpdateSequence(U16(buffer, offset + usaOffset)); + if (replacedBytes != null) + { + // The logged image can carry an older USA than the bytes at this destination. A retained sector + // must fail the fixup check even if the destination already held a partially written page. + var oldTrailers = new HashSet(); + for (var i = 1; i < usaCount; i++) + { + oldTrailers.Add(U16(replacedBytes, (i * stride) - 2)); + } + + while (oldTrailers.Contains(usn)) + { + usn = NextUpdateSequence(usn); + } + } + + Put16(buffer, offset + usaOffset, usn); + for (var i = 1; i < usaCount; i++) + { + var end = offset + (i * stride) - 2; + buffer[offset + usaOffset + (2 * i)] = buffer[end]; + buffer[offset + usaOffset + (2 * i) + 1] = buffer[end + 1]; + Put16(buffer, end, usn); + } + } + + private static ushort NextUpdateSequence(ushort usn) => usn >= 0xFFFE ? (ushort)1 : (ushort)(usn + 1); + + /// Whether the header describes a plausible update sequence array for this length. + public static bool HasFixupLayout(byte[] buffer, int offset, int length) => + TryGetFixupLayout(buffer, offset, length, out _, out _, out _); + + private static bool TryGetFixupLayout(byte[] buffer, int offset, int length, out int usaOffset, out int usaCount, + out int stride) + { + usaOffset = usaCount = stride = 0; + if (offset < 0 || length < 8 || offset > buffer.Length - length) + { + return false; + } + + usaOffset = U16(buffer, offset + 4); + usaCount = U16(buffer, offset + 6); + if (usaCount < 2 || usaOffset < 8 || usaOffset + (2 * usaCount) > length || (usaOffset & 1) != 0) + { + return false; + } + + // NTFS protects every 512 bytes; derive it from the array length so 4K-sector volumes work too. + if (length % (usaCount - 1) != 0) + { + return false; + } + + stride = length / (usaCount - 1); + return stride is >= 256 and <= 4096 && (stride & (stride - 1)) == 0 && usaOffset + (2 * usaCount) <= stride - 2; + } + + public static void ReadExactly(Stream stream, long position, byte[] buffer, int offset, int count) + { + stream.Position = position; + var done = 0; + while (done < count) + { + var n = stream.Read(buffer, offset + done, count - done); + if (n <= 0) + { + throw new JournalException($"Unexpected end of the volume at byte {position + done}"); + } + + done += n; + } + } +} + +/// +/// The parts of the boot sector the journal code needs. +/// +internal sealed class JournalVolumeLayout +{ + private const int MaxClusterSize = 2 * 1024 * 1024; + + public int BytesPerSector { get; private set; } + + public int BytesPerCluster { get; private set; } + + public int MftRecordSize { get; private set; } + + public int IndexBufferSize { get; private set; } + + public long MftCluster { get; private set; } + + public long MftMirrorCluster { get; private set; } + + public long TotalClusters { get; private set; } + + public bool ContainsClusters(long lcn, long count) => lcn > 0 && count > 0 && + lcn < TotalClusters && count <= TotalClusters - lcn; + + public static bool IsRecordSize(long size) => size is >= 256 and <= MaxClusterSize && (size & (size - 1)) == 0; + + public static JournalVolumeLayout Read(Stream volume) + { + var bytes = new byte[512]; + JournalUtilities.ReadExactly(volume, 0, bytes, 0, bytes.Length); + var bpb = BiosParameterBlock.FromBytes(bytes); + if (!bpb.IsValidOemId()) + { + throw new InvalidFileSystemException("BIOS Parameter Block is invalid for an NTFS file system"); + } + + var clusterSize = (long)bpb.BytesPerSector * bpb.SectorsPerCluster; + var mftRecordSize = RecordSize(bpb.RawMftRecordSize, clusterSize); + var indexBufferSize = RecordSize(bpb.RawIndexBufferSize, clusterSize); + if (bpb.TotalSectors16 != 0 || bpb.TotalSectors32 != 0 || bpb.TotalSectors64 <= 0 || + bpb.BytesPerSector is < 256 or > 4096 || (bpb.BytesPerSector & (bpb.BytesPerSector - 1)) != 0 || + bpb.TotalSectors64 > long.MaxValue / bpb.BytesPerSector || bytes[0x0D] is > 0x80 and < (256 - 13) || + clusterSize < bpb.BytesPerSector || clusterSize > MaxClusterSize || (clusterSize & (clusterSize - 1)) != 0 || + !IsRecordSize(mftRecordSize) || !IsRecordSize(indexBufferSize)) + { + throw new JournalException("The NTFS boot sector has invalid volume or record sizes"); + } + + var totalClusters = Math.Min(bpb.TotalSectors64 / bpb.SectorsPerCluster, volume.Length / clusterSize); + if (bpb.MftCluster <= 0 || bpb.MftCluster >= totalClusters) + { + throw new JournalException("The NTFS boot sector places $MFT outside the volume"); + } + + return new JournalVolumeLayout + { + BytesPerSector = bpb.BytesPerSector, + BytesPerCluster = (int)clusterSize, + MftRecordSize = (int)mftRecordSize, + IndexBufferSize = (int)indexBufferSize, + MftCluster = bpb.MftCluster, + MftMirrorCluster = bpb.MftMirrorCluster, + TotalClusters = totalClusters, + }; + } + + private static long RecordSize(byte raw, long clusterSize) => raw <= 0x80 ? raw * clusterSize : + raw >= 256 - 21 ? 1L << (256 - raw) : 0; +} diff --git a/Library/DiscUtils.Ntfs/Journal/LfsLog.cs b/Library/DiscUtils.Ntfs/Journal/LfsLog.cs new file mode 100644 index 000000000..845cbbb2d --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/LfsLog.cs @@ -0,0 +1,719 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.Text; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// A client of the log as listed in the restart area. +/// +internal sealed class LfsClient +{ + public ulong OldestLsn { get; set; } + + public ulong RestartLsn { get; set; } + + public ushort PreviousClient { get; set; } + + public ushort NextClient { get; set; } + + public ushort SequenceNumber { get; set; } + + public string Name { get; set; } +} + +/// +/// One of the two restart pages at the start of $LogFile: the page header and its restart area. +/// +internal sealed class LfsRestartPage +{ + public const ushort NoClient = 0xFFFF; + public const ushort FlagCleanDismount = 0x0002; + + public long Offset { get; set; } + + public uint Signature { get; set; } + + public int SystemPageSize { get; set; } + + public int LogPageSize { get; set; } + + public int RestartAreaOffset { get; set; } + + public short MinorVersion { get; set; } + + public short MajorVersion { get; set; } + + public ulong CurrentLsn { get; set; } + + public ushort LogClients { get; set; } + + public ushort FreeClientList { get; set; } + + public ushort InUseClientList { get; set; } + + public ushort Flags { get; set; } + + public int SequenceNumberBits { get; set; } + + public ushort ClientArrayOffset { get; set; } + + public long FileSize { get; set; } + + public ushort RecordHeaderLength { get; set; } + + public ushort LogPageDataOffset { get; set; } + + public uint OpenLogCount { get; set; } + + public List Clients { get; } = []; + + /// The page with its update sequence array removed. + public byte[] Page { get; set; } + + public bool IsClean => (Flags & FlagCleanDismount) != 0; + + public bool RequiresRecovery => !IsClean && InUseClientList != NoClient; + + public ushort FindNtfsClientIndex() + { + var visited = new bool[Clients.Count]; + var ntfsIndex = NoClient; + ValidateList(InUseClientList, inUse: true); + ValidateList(FreeClientList, inUse: false); + if (ntfsIndex == NoClient) + { + throw new JournalException("$LogFile has no NTFS client in use"); + } + + return ntfsIndex; + + void ValidateList(ushort head, bool inUse) + { + var previous = NoClient; + for (var index = head; index != NoClient; index = Clients[index].NextClient) + { + if (index >= Clients.Count || visited[index] || Clients[index].PreviousClient != previous) + { + throw new JournalException("$LogFile has an invalid LFS client list"); + } + + visited[index] = true; + previous = index; + if (inUse && string.Equals(Clients[index].Name, "NTFS", StringComparison.Ordinal)) + { + if (ntfsIndex != NoClient) + { + throw new JournalException("$LogFile has more than one NTFS client in use"); + } + + ntfsIndex = index; + } + } + } + } + + /// Builds the logical image of the clean LFS 1.1 restart page used by DiscUtils. + public byte[] CreateResetImage(ulong lastLsn) + { + var page = (byte[])Page.Clone(); + var ra = RestartAreaOffset; + Put16(page, 0x1A, 1); + Put16(page, 0x1C, 1); + Put64(page, ra, Math.Max(lastLsn, CurrentLsn)); + Put16(page, ra + 0x0A, 0); + Put16(page, ra + 0x0C, NoClient); + Put16(page, ra + 0x0E, FlagCleanDismount); + Put32(page, ra + 0x28, OpenLogCount + 1); + return page; + } + + public bool MatchesResetOf(LfsRestartPage source) + { + if (!IsClean || InUseClientList != NoClient || MajorVersion != 1 || MinorVersion != 1 || + CurrentLsn < source.CurrentLsn || OpenLogCount != source.OpenLogCount + 1 || Page.Length != source.Page.Length) + { + return false; + } + + var expected = source.CreateResetImage(CurrentLsn); + var usa = U16(expected, 4); + var usaEnd = usa + 2 * U16(expected, 6); + return expected.AsSpan(0, usa).SequenceEqual(Page.AsSpan(0, usa)) && + expected.AsSpan(usaEnd).SequenceEqual(Page.AsSpan(usaEnd)); + } + + /// + /// Reads a restart page at ; null if there is none or it is damaged. + /// + public static LfsRestartPage TryRead(LogFileStorage storage, long offset) + { + var head = new byte[512]; + if (offset > storage.Length - head.Length) + { + return null; + } + + storage.Read(offset, head, 0, head.Length); + var signature = U32(head, 0); + if (signature != SignatureRstr && signature != SignatureChkd) + { + return null; + } + + var systemPageSize = (int)U32(head, 0x10); + var logPageSize = (int)U32(head, 0x14); + if (!IsPageSize(systemPageSize) || !IsPageSize(logPageSize) || + (offset != 0 && offset != systemPageSize) || offset > storage.Length - systemPageSize) + { + return null; + } + + var page = new byte[systemPageSize]; + storage.Read(offset, page, 0, page.Length); + if (!RemoveFixups(page, 0, page.Length)) + { + return null; + } + + var result = new LfsRestartPage + { + Offset = offset, + Signature = signature, + SystemPageSize = systemPageSize, + LogPageSize = logPageSize, + RestartAreaOffset = U16(page, 0x18), + MinorVersion = (short)U16(page, 0x1A), + MajorVersion = (short)U16(page, 0x1C), + Page = page, + }; + + var ra = result.RestartAreaOffset; + if ((ra & 7) != 0 || ra < 0x1E || ra < U16(page, 4) + 2 * U16(page, 6) || ra + 0x30 > systemPageSize) + { + return null; + } + + result.CurrentLsn = U64(page, ra); + result.LogClients = U16(page, ra + 0x08); + result.FreeClientList = U16(page, ra + 0x0A); + result.InUseClientList = U16(page, ra + 0x0C); + result.Flags = U16(page, ra + 0x0E); + result.SequenceNumberBits = (int)U32(page, ra + 0x10); + result.ClientArrayOffset = U16(page, ra + 0x16); + result.FileSize = (long)U64(page, ra + 0x18); + result.RecordHeaderLength = U16(page, ra + 0x24); + result.LogPageDataOffset = U16(page, ra + 0x26); + result.OpenLogCount = U32(page, ra + 0x28); + + var clients = ra + result.ClientArrayOffset; + if (result.LogClients == 0 || result.LogClients > 64 || result.ClientArrayOffset < 0x2C || + clients + (result.LogClients * 0xA0) > systemPageSize) + { + return null; + } + + for (var i = 0; i < result.LogClients; i++) + { + var c = clients + (i * 0xA0); + var nameBytes = (int)Math.Min(U32(page, c + 0x1C), 0x80); + result.Clients.Add(new LfsClient + { + OldestLsn = U64(page, c), + RestartLsn = U64(page, c + 0x08), + PreviousClient = U16(page, c + 0x10), + NextClient = U16(page, c + 0x12), + SequenceNumber = U16(page, c + 0x14), + Name = Encoding.Unicode.GetString(page, c + 0x20, nameBytes), + }); + } + + if (result.InUseClientList != NoClient && result.InUseClientList >= result.LogClients) + { + return null; + } + + return result; + } + + private static bool IsPageSize(int size) => size is >= 512 and <= 0x10000 && (size & (size - 1)) == 0; +} + +/// +/// A log record as LFS stores it: its header and the client's data. +/// +internal sealed class LfsRecord +{ + public const uint TypeClientRecord = 1; + public const uint TypeClientRestart = 2; + + public ulong Lsn { get; set; } + + public ulong ClientPreviousLsn { get; set; } + + public ulong ClientUndoNextLsn { get; set; } + + public ushort ClientSequenceNumber { get; set; } + + public ushort ClientIndex { get; set; } + + public uint RecordType { get; set; } + + public uint TransactionId { get; set; } + + public ushort Flags { get; set; } + + public byte[] ClientData { get; set; } +} + +/// +/// Reads the log records of $LogFile (log file service versions 1.x and 2.0). +/// +/// +/// LFS 2.0 (Windows 8 and later, while the volume is mounted) keeps 32 "tail" copies of recently written log pages +/// in pages 2 to 0x21, version 1.x keeps two in pages 2 and 3. A copy can be newer than the page at its place in +/// the log, so every page is read from whichever copy holds the highest LSN. A record belongs to the log only if +/// its header carries exactly the LSN it was looked up by; the first position where that fails is the end. +/// +internal sealed class LfsLog +{ + // The NTFS client uses 16-bit counts for the LCN array and 16-bit offsets and lengths for operation data. + private const int MaxClientDataLength = NtfsLogRecord.HeaderLength + 8 * ushort.MaxValue; + + private readonly LogFileStorage _storage; + private readonly ushort _ntfsClientIndex; + private readonly Dictionary _pages = []; + private readonly Dictionary> _tailCopies = []; + + private LfsLog(LogFileStorage storage, LfsRestartPage restart, LfsRestartPage publishedReset, ushort ntfsClientIndex) + { + _storage = storage; + _ntfsClientIndex = ntfsClientIndex; + Restart = restart; + PublishedReset = publishedReset; + PageSize = restart.LogPageSize; + SequenceNumberBits = restart.SequenceNumberBits; + FileSize = restart.FileSize; + DataOffset = restart.LogPageDataOffset; + RecordHeaderLength = restart.RecordHeaderLength; + FirstPageOffset = (restart.MajorVersion >= 2 ? 0x22L : 4L) * PageSize; + ReadTailCopies(); + } + + public LogFileStorage Storage => _storage; + + /// The restart page used for analysis. + public LfsRestartPage Restart { get; } + + public LfsClient NtfsClient => _ntfsClientIndex == LfsRestartPage.NoClient ? null : Restart.Clients[_ntfsClientIndex]; + + /// A clean reset image whose sibling is still the source of that reset. + public LfsRestartPage PublishedReset { get; } + + public int PageSize { get; } + + public int SequenceNumberBits { get; } + + public long FileSize { get; } + + public int DataOffset { get; } + + public int RecordHeaderLength { get; } + + public long FirstPageOffset { get; } + + /// + /// The kind of $LogFile found. + /// + public enum Kind + { + /// No restart page; the file is filled with 0xFF (freshly formatted or reset). Nothing to do. + Empty, + + /// chkdsk wrote a CHKD restart page: it already repaired the volume and obsoleted the log. + CheckedByChkdsk, + + /// A usable log. + Log, + } + + /// + /// Opens the log. tells whether there is a log at all; the result is null unless + /// it is . + /// + public static LfsLog Open(LogFileStorage storage, out Kind kind) + { + var first = LfsRestartPage.TryRead(storage, 0); + LfsRestartPage second = null; + if (first != null) + { + second = LfsRestartPage.TryRead(storage, first.SystemPageSize); + } + else + { + for (long offset = 512; offset <= 0x10000 && second == null; offset *= 2) + { + second = LfsRestartPage.TryRead(storage, offset); + } + } + + if (first == null && second == null) + { + var head = new byte[512]; + storage.Read(0, head, 0, head.Length); + if (Array.TrueForAll(head, b => b == 0xFF)) + { + kind = Kind.Empty; + return null; + } + + throw new JournalException("$LogFile has no valid restart page"); + } + + if (first?.Signature == SignatureChkd || second?.Signature == SignatureChkd) + { + kind = Kind.CheckedByChkdsk; + return null; + } + + var restart = first == null || (second != null && second.CurrentLsn > first.CurrentLsn) ? second : first; + LfsRestartPage publishedReset = null; + if (first != null && second != null && first.RequiresRecovery != second.RequiresRecovery) + { + var dirty = first.RequiresRecovery ? first : second; + var clean = first.RequiresRecovery ? second : first; + if (clean.MatchesResetOf(dirty)) + { + // Finish an interrupted DiscUtils reset only after rechecking the metadata against the old log. + restart = dirty; + publishedReset = clean; + } + else if (first.CurrentLsn == second.CurrentLsn) + { + throw new JournalException("The $LogFile restart pages disagree about recovery at the same LSN"); + } + } + if (restart.MajorVersion is not (1 or 2)) + { + throw new JournalException( + $"$LogFile uses log file service version {restart.MajorVersion}.{restart.MinorVersion}, which DiscUtils does not support"); + } + + var minLogPages = restart.MajorVersion >= 2 ? 0x23L : 8L; + if (restart.SequenceNumberBits is < 16 or > 60 || restart.FileSize > storage.Length || + restart.FileSize < minLogPages * restart.LogPageSize || restart.FileSize < 2L * restart.SystemPageSize || + restart.FileSize > (1L << (67 - restart.SequenceNumberBits)) || (restart.FileSize & (restart.LogPageSize - 1)) != 0 || + restart.RecordHeaderLength != 0x30 || restart.LogPageDataOffset < 0x28 || + restart.LogPageDataOffset > restart.LogPageSize - restart.RecordHeaderLength || (restart.LogPageDataOffset & 7) != 0) + { + throw new JournalException("$LogFile has an implausible restart area"); + } + + var ntfsClientIndex = LfsRestartPage.NoClient; + if (restart.RequiresRecovery) + { + ntfsClientIndex = restart.FindNtfsClientIndex(); + var other = restart == first ? second : first; + if (other?.RequiresRecovery == true && restart.CurrentLsn == other.CurrentLsn) + { + var otherIndex = other.FindNtfsClientIndex(); + var client = restart.Clients[ntfsClientIndex]; + var otherClient = other.Clients[otherIndex]; + if (restart.LogClients != other.LogClients || restart.InUseClientList != other.InUseClientList || + ntfsClientIndex != otherIndex || client.SequenceNumber != otherClient.SequenceNumber || + client.RestartLsn != otherClient.RestartLsn || client.OldestLsn != otherClient.OldestLsn) + { + throw new JournalException("The $LogFile restart pages disagree about the checkpoint at the same LSN"); + } + } + } + + kind = Kind.Log; + return new LfsLog(storage, restart, publishedReset, ntfsClientIndex); + } + + public bool IsForNtfsClient(LfsRecord record) + { + if (record.ClientIndex >= Restart.LogClients) + { + throw new JournalException($"Log record {record.Lsn:X} has client index {record.ClientIndex} outside the restart client array"); + } + + return record.ClientIndex == _ntfsClientIndex && record.ClientSequenceNumber == NtfsClient.SequenceNumber; + } + + public void RequireNtfsRecord(LfsRecord record, string role, uint? expectedType = null) + { + if (!IsForNtfsClient(record)) + { + throw new JournalException($"The {role} at {record.Lsn:X} does not belong to the active NTFS client"); + } + + if (record.RecordType is not (LfsRecord.TypeClientRecord or LfsRecord.TypeClientRestart) || + (expectedType.HasValue && record.RecordType != expectedType.Value)) + { + throw new JournalException($"The {role} at {record.Lsn:X} has the wrong LFS record type"); + } + } + + public long LsnToOffset(ulong lsn) => (long)((lsn << SequenceNumberBits) >> (SequenceNumberBits - 3)); + + public ulong SequenceOf(ulong lsn) => lsn >> (64 - SequenceNumberBits); + + private ulong MakeLsn(ulong sequence, long offset) => (sequence << (64 - SequenceNumberBits)) + (ulong)(offset >> 3); + + /// + /// Reads the record with this LSN; null if the log holds no such record (the end of the log). + /// Set to false to check the identity of a retained header alone. + /// + public LfsRecord ReadRecord(ulong lsn, bool readClientData = true) + { + if (lsn == 0) + { + return null; + } + + var offset = LsnToOffset(lsn); + var pageOffset = offset & ~(long)(PageSize - 1); + var inPage = (int)(offset - pageOffset); + if (pageOffset < FirstPageOffset || offset >= FileSize || inPage < DataOffset || inPage + RecordHeaderLength > PageSize) + { + return null; + } + + var page = GetPage(pageOffset, lsn); + if (page == null || U64(page, inPage) != lsn) + { + return null; + } + + var record = new LfsRecord + { + Lsn = lsn, + ClientPreviousLsn = U64(page, inPage + 0x08), + ClientUndoNextLsn = U64(page, inPage + 0x10), + ClientSequenceNumber = U16(page, inPage + 0x1C), + ClientIndex = U16(page, inPage + 0x1E), + RecordType = U32(page, inPage + 0x20), + TransactionId = U32(page, inPage + 0x24), + Flags = U16(page, inPage + 0x28), + }; + + if (!readClientData) + { + return record; + } + + var dataLength = U32(page, inPage + 0x18); + var firstPageCapacity = PageSize - (inPage + RecordHeaderLength); + var continuationPages = (FileSize - FirstPageOffset) / PageSize - 1; + if (dataLength > MaxClientDataLength || dataLength > firstPageCapacity + continuationPages * (PageSize - DataOffset)) + { + throw new JournalException($"Log record {lsn:X} has an invalid client data length"); + } + + record.ClientData = new byte[(int)dataLength]; + var copied = Math.Min(record.ClientData.Length, firstPageCapacity); + Array.Copy(page, inPage + RecordHeaderLength, record.ClientData, 0, copied); + var current = pageOffset; + while (copied < record.ClientData.Length) + { + current = NextPageOffset(current, out _); + var next = GetPage(current, lsn); + if (next == null) + { + return null; + } + + var chunk = Math.Min(record.ClientData.Length - copied, PageSize - DataOffset); + Array.Copy(next, DataOffset, record.ClientData, copied, chunk); + copied += chunk; + } + + return record; + } + + /// + /// The LSN right after a record: records are 8-byte aligned, continue after the header of the next page, and + /// a record whose header no longer fits in a page starts on the next page. + /// + public ulong NextLsn(LfsRecord record) + { + var offset = LsnToOffset(record.Lsn); + var sequence = SequenceOf(record.Lsn); + var remaining = (long)Align8(RecordHeaderLength + record.ClientData.Length); + var pageOffset = offset & ~(long)(PageSize - 1); + var room = PageSize - (offset - pageOffset); + long position; + if (remaining < room) + { + position = offset + remaining; + } + else + { + remaining -= room; + while (true) + { + pageOffset = NextPageOffset(pageOffset, out var wrapped); + if (wrapped) + { + sequence++; + } + + var available = PageSize - DataOffset; + if (remaining < available) + { + position = pageOffset + DataOffset + remaining; + break; + } + + remaining -= available; + } + } + + if (PageSize - (position & (PageSize - 1)) < RecordHeaderLength) + { + pageOffset = NextPageOffset(position & ~(long)(PageSize - 1), out var wrapped); + if (wrapped) + { + sequence++; + } + + position = pageOffset + DataOffset; + } + + return MakeLsn(sequence, position); + } + + /// + /// Reads the records from up to the end of the log, in order. + /// + public IEnumerable ReadForward(ulong startLsn) + { + var record = ReadRecord(startLsn); + while (record != null) + { + yield return record; + var next = NextLsn(record); + if (next <= record.Lsn) + { + throw new JournalException($"The next LSN after {record.Lsn:X} overflows the log sequence"); + } + + record = ReadRecord(next); + } + } + + private long NextPageOffset(long pageOffset, out bool wrapped) + { + var next = pageOffset + PageSize; + wrapped = next >= FileSize; + return wrapped ? FirstPageOffset : next; + } + + /// + /// The log page at from whichever copy (in place or tail copy) is the newest + /// one that can hold data of the record ; null if none is valid. + /// + private byte[] GetPage(long pageOffset, ulong lsn) + { + if (!_pages.TryGetValue(pageOffset, out var main)) + { + main = ReadLogPage(pageOffset); + _pages[pageOffset] = main; + } + + byte[] best = null; + ulong bestLsn = 0; + if (main != null) + { + best = main; + bestLsn = U64(main, 0x08); + } + + if (_tailCopies.TryGetValue(pageOffset, out var copies)) + { + foreach (var copy in copies) + { + var copyLsn = TailCopyLsn(copy); + if (best == null || copyLsn > bestLsn) + { + best = copy; + bestLsn = copyLsn; + } + } + } + + // A page that was last written before this record existed cannot hold any of its data. + if (best != null && bestLsn < lsn) + { + return null; + } + + return best; + } + + private ulong TailCopyLsn(byte[] copy) => Restart.MajorVersion >= 2 ? U64(copy, 0x08) : U64(copy, 0x20); + + private byte[] ReadLogPage(long pageOffset) + { + var page = new byte[PageSize]; + _storage.Read(pageOffset, page, 0, page.Length); + if (U32(page, 0) != SignatureRcrd || !RemoveFixups(page, 0, page.Length)) + { + return null; + } + + return page; + } + + private void ReadTailCopies() + { + for (var offset = 2L * PageSize; offset < FirstPageOffset; offset += PageSize) + { + var page = ReadLogPage(offset); + if (page == null) + { + continue; + } + + // Version 2.0 keeps the place of the copied page at 0x3C; version 1.x puts it where an LSN would be. + var target = Restart.MajorVersion >= 2 ? U32(page, 0x3C) : (long)U64(page, 0x08); + if (target < FirstPageOffset || target >= FileSize || (target & (PageSize - 1)) != 0) + { + continue; + } + + if (!_tailCopies.TryGetValue(target, out var list)) + { + list = []; + _tailCopies[target] = list; + } + + list.Add(page); + } + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/LogFileStorage.cs b/Library/DiscUtils.Ntfs/Journal/LogFileStorage.cs new file mode 100644 index 000000000..860d39061 --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/LogFileStorage.cs @@ -0,0 +1,335 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// Byte access to the $LogFile data on the raw volume, located through MFT record 2 before the file system is +/// mounted. $LogFile's own record is written at format time only, so it is never stale. +/// +internal sealed class LogFileStorage +{ + private const int LogFileRecordNumber = 2; + private const uint AttributeList = 0x20; + private const uint AttributeData = 0x80; + private const uint AttributeEnd = 0xFFFFFFFF; + + private readonly Stream _volume; + private readonly int _clusterSize; + private readonly List<(long Vcn, long Lcn, long Length)> _extents; + + private LogFileStorage(Stream volume, int clusterSize, List<(long, long, long)> extents, long length) + { + _volume = volume; + _clusterSize = clusterSize; + _extents = extents; + Length = length; + } + + /// Gets the size of the $LogFile data in bytes. + public long Length { get; } + + public static LogFileStorage Open(Stream volume, JournalVolumeLayout layout) + { + var record = ReadLogFileRecord(volume, layout, layout.MftCluster) + ?? ReadLogFileRecord(volume, layout, layout.MftMirrorCluster) + ?? throw new JournalException("Neither $MFT nor $MFTMirr holds a valid record for $LogFile"); + + var (extents, length) = ReadDataAttribute(record, layout, "$LogFile"); + var mftPrefixClusters = (4L * layout.MftRecordSize + layout.BytesPerCluster - 1) / layout.BytesPerCluster; + foreach (var extent in extents) + { + if (Overlaps(extent, layout.MftCluster, mftPrefixClusters) || + (layout.ContainsClusters(layout.MftMirrorCluster, mftPrefixClusters) && + Overlaps(extent, layout.MftMirrorCluster, mftPrefixClusters))) + { + throw new JournalException("$LogFile overlaps $MFT or $MFTMirr"); + } + } + + return new LogFileStorage(volume, layout.BytesPerCluster, extents, length); + } + + private static bool Overlaps((long Vcn, long Lcn, long Length) extent, long start, long count) => + extent.Lcn < start + count && start < extent.Lcn + extent.Length; + + public bool ContainsCluster(long lcn) => _extents.Any(extent => lcn >= extent.Lcn && lcn - extent.Lcn < extent.Length); + + // Both $LogFile and the supported $MFTMirr layout can be checked from a single base record. + internal static (List<(long Vcn, long Lcn, long Length)> Extents, long Length) ReadDataAttribute( + byte[] record, JournalVolumeLayout layout, string fileName) + { + var used = U32(record, 0x18); + var firstAttribute = U16(record, 0x14); + if (U64(record, 0x20) != 0 || used > record.Length || firstAttribute < 0x2A || used < firstAttribute + 4) + { + throw new JournalException($"{fileName} has a damaged base file record"); + } + + var data = -1; + var dataAttributeLength = 0; + var end = (int)used; + for (var offset = (int)firstAttribute; offset <= end - 4;) + { + var type = U32(record, offset); + if (type == AttributeEnd) + { + if (data < 0) + { + throw new JournalException($"{fileName} has no unnamed data attribute"); + } + + return ReadNonResidentData(record, data, dataAttributeLength, layout, fileName); + } + + if (offset > end - 8) + { + break; + } + + var length = U32(record, offset + 4); + if (length < 0x18 || length > end - offset || (length & 7) != 0) + { + break; + } + + if (type == AttributeList) + { + throw new JournalException($"{fileName} has an $ATTRIBUTE_LIST, which DiscUtils does not support for replay"); + } + + if (type == AttributeData && record[offset + 9] == 0) + { + if (data >= 0) + { + throw new JournalException($"{fileName} has split unnamed data, which DiscUtils does not support for replay"); + } + + data = offset; + dataAttributeLength = (int)length; + } + + offset += (int)length; + } + + throw new JournalException($"{fileName} has a damaged attribute list in its base file record"); + } + + private static (List<(long Vcn, long Lcn, long Length)> Extents, long Length) ReadNonResidentData( + byte[] record, int offset, int length, JournalVolumeLayout layout, string fileName) + { + if (record[offset + 8] != 1 || length < 0x40) + { + throw new JournalException($"{fileName} does not have a complete non-resident data header"); + } + + if (U64(record, offset + 0x10) != 0) + { + throw new JournalException($"{fileName} has a nonzero starting VCN, which DiscUtils does not support for replay"); + } + + if (U16(record, offset + 0x0C) != 0 || U16(record, offset + 0x22) != 0) + { + throw new JournalException($"{fileName} has unsupported data attribute flags or compression"); + } + + var runsOffset = U16(record, offset + 0x20); + if (runsOffset < 0x40 || runsOffset >= length) + { + throw new JournalException($"{fileName} has an invalid mapping pairs offset"); + } + + var allocated = (long)U64(record, offset + 0x28); + var dataLength = (long)U64(record, offset + 0x30); + var initialized = (long)U64(record, offset + 0x38); + if (dataLength <= 0 || dataLength > allocated || initialized != dataLength || allocated % layout.BytesPerCluster != 0) + { + throw new JournalException($"{fileName} has invalid allocated, data or initialized lengths"); + } + + var extents = ParseRuns(record, offset + runsOffset, offset + length, layout.TotalClusters, fileName); + var last = extents[extents.Count - 1]; + var clusters = last.Vcn + last.Length; + if (U64(record, offset + 0x18) != (ulong)(clusters - 1) || clusters != allocated / layout.BytesPerCluster) + { + throw new JournalException($"{fileName} run list does not cover the declared allocation in one attribute extent"); + } + + return (extents, dataLength); + } + + private static byte[] ReadLogFileRecord(Stream volume, JournalVolumeLayout layout, long mftCluster) + { + var end = (LogFileRecordNumber + 1L) * layout.MftRecordSize; + if (!layout.ContainsClusters(mftCluster, (end + layout.BytesPerCluster - 1) / layout.BytesPerCluster)) + { + return null; + } + + var record = new byte[layout.MftRecordSize]; + var position = (mftCluster * layout.BytesPerCluster) + (end - record.Length); + ReadExactly(volume, position, record, 0, record.Length); + if (U32(record, 0) != SignatureFile || !RemoveFixups(record, 0, record.Length) || (U16(record, 0x16) & 1) == 0) + { + return null; + } + + return record; + } + + internal static List<(long Vcn, long Lcn, long Length)> ParseRuns(byte[] record, int offset, int end, + long totalClusters, string fileName) + { + var extents = new List<(long Vcn, long Lcn, long Length)>(); + long vcn = 0; + long lcn = 0; + while (offset < end && record[offset] != 0) + { + var header = record[offset]; + var lengthSize = header & 0x0F; + var offsetSize = header >> 4; + if (lengthSize is 0 or > 8 || offsetSize > 8 || offset + 1 + lengthSize + offsetSize > end) + { + throw new JournalException($"{fileName} has a damaged run list"); + } + + var runLength = ReadUnsigned(record, offset + 1, lengthSize); + if (offsetSize == 0) + { + throw new JournalException($"{fileName} has a sparse run"); + } + + var delta = ReadSigned(record, offset + 1 + lengthSize, offsetSize); + if (delta <= -lcn || delta > long.MaxValue - lcn) + { + throw new JournalException($"{fileName} run has an invalid or overflowing LCN"); + } + + lcn += delta; + if (runLength <= 0 || runLength > totalClusters - lcn || runLength > totalClusters - vcn) + { + throw new JournalException($"{fileName} run has an invalid length or extends outside the volume"); + } + + extents.Add((vcn, lcn, runLength)); + vcn += runLength; + offset += 1 + lengthSize + offsetSize; + } + + if (extents.Count == 0 || offset == end) + { + throw new JournalException($"{fileName} run list is empty or has no terminator"); + } + + long previousEnd = 0; + foreach (var extent in extents.OrderBy(extent => extent.Lcn)) + { + if (extent.Lcn < previousEnd) + { + throw new JournalException($"{fileName} has overlapping physical runs"); + } + + previousEnd = extent.Lcn + extent.Length; + } + + return extents; + } + + private static long ReadUnsigned(byte[] buffer, int offset, int size) + { + long value = 0; + for (var i = size - 1; i >= 0; i--) + { + value = (value << 8) | buffer[offset + i]; + } + + return value; + } + + private static long ReadSigned(byte[] buffer, int offset, int size) + { + var value = ReadUnsigned(buffer, offset, size); + + // Sign-extend from the top bit of the last byte. + if (size < 8 && (buffer[offset + size - 1] & 0x80) != 0) + { + value |= -1L << (size * 8); + } + + return value; + } + + public void Read(long position, byte[] buffer, int offset, int count) => Transfer(position, buffer, offset, count, write: false); + + public void Write(long position, byte[] buffer, int offset, int count) => Transfer(position, buffer, offset, count, write: true); + + private void Transfer(long position, byte[] buffer, int offset, int count, bool write) + { + if (position < 0 || count < 0 || position > Length - count) + { + throw new JournalException($"Access beyond the end of $LogFile at {position}"); + } + + while (count > 0) + { + var vcn = position / _clusterSize; + var inCluster = (int)(position % _clusterSize); + var extent = FindExtent(vcn); + var available = ((extent.Vcn + extent.Length - vcn) * _clusterSize) - inCluster; + var chunk = (int)Math.Min(count, available); + var physical = ((extent.Lcn + (vcn - extent.Vcn)) * _clusterSize) + inCluster; + if (write) + { + _volume.Position = physical; + _volume.Write(buffer, offset, chunk); + } + else + { + ReadExactly(_volume, physical, buffer, offset, chunk); + } + + position += chunk; + offset += chunk; + count -= chunk; + } + } + + private (long Vcn, long Lcn, long Length) FindExtent(long vcn) + { + // The extents follow each other from VCN 0, so the first one that ends after the VCN holds it. + foreach (var extent in _extents) + { + if (vcn < extent.Vcn + extent.Length) + { + return extent; + } + } + + throw new JournalException($"$LogFile has no cluster for VCN {vcn}"); + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsHibernation.cs b/Library/DiscUtils.Ntfs/Journal/NtfsHibernation.cs new file mode 100644 index 000000000..6e3dc813a --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsHibernation.cs @@ -0,0 +1,266 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DiscUtils.Streams; +using DiscUtils.Streams.Compatibility; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +internal enum NtfsHibernationEvidence +{ + NoLocalEvidence, + Active, + Indeterminate, +} + +/// Checks the local hibernation file without mounting for writes or running journal recovery. +internal static class NtfsHibernation +{ + private const int HeaderSize = 4096; + private const string FileName = "hiberfil.sys"; + + public static (NtfsHibernationEvidence Evidence, string Detail) Probe(Stream volume, JournalVolumeLayout layout, + bool allowMftMirror = false) + { + try + { + using var readOnly = new JournalReadOnlyStream(volume); + var boot = new byte[512]; + ReadExactly(readOnly, 0, boot, 0, boot.Length); + var context = new NtfsContext + { + RawStream = readOnly, + ReadOnly = true, + BiosParameterBlock = BiosParameterBlock.FromBytes(boot), + Options = new NtfsOptions(), + }; + + using var mft = new MasterFileTable(context); + context.Mft = mft; + mft.Initialize(new File(context, ReadMftBootstrap(readOnly, layout, allowMftMirror))); + var root = ReadRootDirectory(context); + var entry = root.GetEntryByName(FileName, requireUnique: true); + if (entry == null) + { + return (NtfsHibernationEvidence.NoLocalEvidence, "No root hiberfil.sys was found"); + } + + var record = mft.GetRecord(entry.Value.Reference); + if (record == null || record.BaseFile.Value != 0 || + record.SequenceNumber != entry.Value.Reference.SequenceNumber || + (record.Flags & (FileRecordFlags.InUse | FileRecordFlags.IsDirectory)) != FileRecordFlags.InUse) + { + throw new InvalidDataException("The root hiberfil.sys entry does not reference an allocated file record"); + } + + var file = new File(context, record); + if (file.GetAttribute(AttributeType.ReparsePoint, null) != null || + !file.GetStreams(AttributeType.FileName, null).Any(stream => + { + if (stream.Attribute.IsNonResident || stream.Attribute.Length is < 0x42 or > 0x240) + { + throw new InvalidDataException("hiberfil.sys has an invalid file name attribute"); + } + + var name = stream.GetContent(); + return name.ParentDirectory == root.MftReference && + string.Equals(name.FileName, FileName, StringComparison.OrdinalIgnoreCase); + })) + { + throw new InvalidDataException("The root hiberfil.sys entry has an inconsistent file record"); + } + + var data = file.GetAttribute(AttributeType.Data, null); + if (data?.PrimaryRecord is not NonResidentAttributeRecord first || first.Flags != AttributeFlags.None || + first.StartVcn != 0 || first.DataLength < HeaderSize || first.InitializedDataLength < HeaderSize || + first.InitializedDataLength > first.DataLength || first.DataLength > first.AllocatedLength || + first.AllocatedLength % layout.BytesPerCluster != 0 || first.AllocatedLength / layout.BytesPerCluster > layout.TotalClusters) + { + throw new InvalidDataException("hiberfil.sys has no complete initialized header in ordinary file data"); + } + + var header = ReadHeader(readOnly, layout, first); + var signature = header.AsSpan(0, 4); + if (signature.SequenceEqual("hibr"u8) || signature.SequenceEqual("HIBR"u8) || signature.SequenceEqual("HORM"u8)) + { + return (NtfsHibernationEvidence.Active, "hiberfil.sys has an active Windows hibernation signature"); + } + + if (signature.SequenceEqual("wake"u8) || signature.SequenceEqual("WAKE"u8)) + { + return (NtfsHibernationEvidence.NoLocalEvidence, "The local hiberfil.sys has an invalidated resume signature"); + } + + if (Array.TrueForAll(header, b => b == 0)) + { + return (NtfsHibernationEvidence.NoLocalEvidence, "The local hiberfil.sys header is zeroed"); + } + + if (signature.SequenceEqual("rstr"u8) || signature.SequenceEqual("RSTR"u8)) + { + return (NtfsHibernationEvidence.Indeterminate, "hiberfil.sys has a resume-in-progress signature"); + } + + return (NtfsHibernationEvidence.Indeterminate, "hiberfil.sys has an unrecognized nonzero header"); + } + catch (Exception e) when (e is IOException or InvalidDataException or ArgumentException or IndexOutOfRangeException or + OverflowException or InvalidOperationException or NotSupportedException or NotImplementedException or + NullReferenceException) + { + return (NtfsHibernationEvidence.Indeterminate, e.Message); + } + } + + private static FileRecord ReadMftBootstrap(Stream volume, JournalVolumeLayout layout, bool allowMftMirror) + { + var bytes = new byte[layout.MftRecordSize]; + ReadRecord(layout.MftCluster); + if (allowMftMirror && !RemoveFixups((byte[])bytes.Clone(), 0, bytes.Length)) + { + // Only the bootstrap record comes from the mirror. The lookup still reads the original volume. + ReadRecord(layout.MftMirrorCluster); + } + + var record = new FileRecord(layout.BytesPerSector); + record.FromBytes(bytes); + if (record.MasterFileTableIndex != MasterFileTable.MftIndex || record.BaseFile.Value != 0 || + (record.Flags & FileRecordFlags.InUse) == 0 || + record.GetAttribute(AttributeType.Data) == null || record.GetAttribute(AttributeType.Bitmap) == null) + { + throw new InvalidDataException("The $MFT bootstrap record is invalid"); + } + + return record; + + void ReadRecord(long lcn) + { + if (!layout.ContainsClusters(lcn, (bytes.Length + (long)layout.BytesPerCluster - 1) / layout.BytesPerCluster)) + { + throw new InvalidDataException("The $MFT bootstrap record is outside the volume"); + } + + ReadExactly(volume, lcn * layout.BytesPerCluster, bytes, 0, bytes.Length); + if (U32(bytes, 0) != SignatureFile || !HasFixupLayout(bytes, 0, bytes.Length)) + { + throw new InvalidDataException("The $MFT bootstrap record is invalid"); + } + } + } + + private static Directory ReadRootDirectory(NtfsContext context) + { + var record = context.Mft.GetRecord(MasterFileTable.RootDirIndex, ignoreMagic: false); + const FileRecordFlags rootFlags = FileRecordFlags.InUse | FileRecordFlags.IsDirectory; + if (record == null || record.BaseFile.Value != 0 || (record.Flags & rootFlags) != rootFlags) + { + throw new InvalidDataException("The root directory record is unavailable"); + } + + var root = new Directory(context, record); + if (root.GetAttribute(AttributeType.IndexRoot, "$I30") is not { IsNonResident: false, Flags: AttributeFlags.None }) + { + throw new InvalidDataException("The root directory index is unavailable"); + } + + var upCaseRecord = context.Mft.GetRecord(MasterFileTable.UpCaseIndex, ignoreMagic: false); + if (upCaseRecord == null || upCaseRecord.BaseFile.Value != 0 || (upCaseRecord.Flags & FileRecordFlags.InUse) == 0) + { + throw new InvalidDataException("The $UpCase record is unavailable"); + } + + var upCase = new File(context, upCaseRecord); + if (upCase.GetAttribute(AttributeType.Data, null) is not { Flags: AttributeFlags.None } data || + data.Length is < 512 or > 131072 || (data.Length & 1) != 0 || data.PrimaryRecord.InitializedDataLength != data.Length) + { + throw new InvalidDataException("The $UpCase table is invalid"); + } + + context.UpperCase = new UpperCase(upCase); + if (context.UpperCase.Compare(FileName, "HIBERFIL.SYS") != 0) + { + throw new InvalidDataException("The $UpCase table cannot compare the hibernation file name"); + } + + return root; + } + + private static byte[] ReadHeader(Stream volume, JournalVolumeLayout layout, NonResidentAttributeRecord extent) + { + // A sparse or uninitialized range can read back as zeros without any bytes having been checked on disk. + var header = new byte[HeaderSize]; + var done = 0; + long lcn = 0; + long vcn = 0; + foreach (var run in extent.DataRuns) + { + lcn = checked(lcn + run.RunOffset); + if (run.IsSparse || !layout.ContainsClusters(lcn, run.RunLength) || + vcn > extent.LastVcn || run.RunLength - 1 > extent.LastVcn - vcn) + { + throw new InvalidDataException("hiberfil.sys has an invalid header cluster mapping"); + } + + var count = (int)Math.Min(run.RunLength * layout.BytesPerCluster, header.Length - done); + ReadExactly(volume, lcn * layout.BytesPerCluster, header, done, count); + done += count; + vcn += run.RunLength; + if (done == header.Length) + { + return header; + } + } + + throw new InvalidDataException("hiberfil.sys does not map a complete header in its first extent"); + } +} + +/// A borrowed volume stream that cannot write, flush or close its source. +internal sealed class JournalReadOnlyStream(Stream volume) : SparseStream.ReadOnlySparseStream +{ + public override bool CanRead => volume.CanRead; + + public override bool CanSeek => volume.CanSeek; + + public override long Length => volume.Length; + + public override long Position + { + get => volume.Position; + set => volume.Position = value; + } + + public override IEnumerable Extents => volume is SparseStream sparse ? sparse.Extents : [new StreamExtent(0, Length)]; + + public override long? GetPositionInBaseStream(Stream baseStream, long virtualPosition) => + ReferenceEquals(this, baseStream) ? virtualPosition : volume.GetPositionInBaseStream(baseStream, virtualPosition); + + public override long Seek(long offset, SeekOrigin origin) => volume.Seek(offset, origin); + + public override int Read(byte[] buffer, int offset, int count) => volume.Read(buffer, offset, count); + + public override int Read(Span buffer) => volume.Read(buffer); +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsLogActions.cs b/Library/DiscUtils.Ntfs/Journal/NtfsLogActions.cs new file mode 100644 index 000000000..1bcedf3f9 --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsLogActions.cs @@ -0,0 +1,547 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// The byte-level meaning of every operation NTFS logs, applied to a page in its logical form. Each action checks +/// that the page looks the way the log record expects (an attribute or index entry starts exactly at the logged +/// offset, sizes stay inside the record) and throws otherwise, so a replay never writes a guess. +/// +internal static class NtfsLogActions +{ + // File record header + private const int FrSequence = 0x10; + private const int FrHardLinks = 0x12; + private const int FrFirstAttribute = 0x14; + private const int FrFlags = 0x16; + private const int FrBytesInUse = 0x18; + private const int FrNextAttributeId = 0x28; + + // Attribute header + private const int AType = 0x00; + private const int ALength = 0x04; + private const int ANonResident = 0x08; + private const int AId = 0x0E; + private const int RValueLength = 0x10; + private const int RValueOffset = 0x14; + private const int RIndexed = 0x16; + private const int NStartVcn = 0x10; + private const int NLastVcn = 0x18; + private const int NRunsOffset = 0x20; + private const int NAllocated = 0x28; + private const int NDataSize = 0x30; + private const int NValidSize = 0x38; + private const int NTotalSize = 0x40; + + private const uint AttributeEnd = 0xFFFFFFFF; + private const uint AttributeIndexRoot = 0x90; + + // Index header (inside an INDEX_ROOT value at +0x10, inside an INDX buffer at +0x18) + private const int IhEntriesOffset = 0x00; + private const int IhUsed = 0x04; + private const int IhTotal = 0x08; + private const int IndxHeader = 0x18; + + // Index entry + private const int IeLength = 0x08; + private const int FileNameDuplicatedInfo = 0x18; + private const int DuplicatedInfoLength = 0x38; + + /// + /// Applies with to a file record. + /// + public static void ApplyToFileRecord(NtfsLogOperation op, NtfsLogRecord record, byte[] p, ArraySegment data) + { + var roff = record.RecordOffset; + var aoff = record.AttributeOffset; + var size = p.Length; + switch (op) + { + case NtfsLogOperation.InitializeFileRecordSegment: + // The record image; like Windows, the rest of the record keeps its old bytes. + Require(roff + data.Count <= size, record, "the new file record does not fit"); + Copy(data, p, roff); + return; + + case NtfsLogOperation.DeallocateFileRecordSegment: + Put16(p, FrFlags, (ushort)(U16(p, FrFlags) & ~1)); + Put16(p, FrSequence, (ushort)(U16(p, FrSequence) + 1)); + return; + + case NtfsLogOperation.WriteEndOfFileRecordSegment: + RequireAttributeAt(p, roff, record, allowEnd: true); + Require(roff + data.Count <= size, record, "the end of the file record does not fit"); + Copy(data, p, roff); + Put32(p, FrBytesInUse, (uint)Align8(roff + data.Count)); + return; + + case NtfsLogOperation.CreateAttribute: + CreateAttribute(p, record, data); + return; + + case NtfsLogOperation.DeleteAttribute: + DeleteAttribute(p, record); + return; + + case NtfsLogOperation.UpdateResidentValue: + UpdateResidentValue(p, record, data); + return; + + case NtfsLogOperation.UpdateMappingPairs: + UpdateMappingPairs(p, record, data); + return; + + case NtfsLogOperation.SetNewAttributeSizes: + RequireAttributeAt(p, roff, record); + Require(p[roff + ANonResident] != 0, record, "new sizes for a resident attribute"); + var length = U32(p, roff + ALength); + Require(length >= NValidSize + 8, record, "the non-resident attribute header is too short"); + var runsOffset = U16(p, roff + NRunsOffset); + Require(runsOffset >= NValidSize + 8 && runsOffset <= length, record, "the mapping pairs offset is invalid"); + Require(data.Count >= 24, record, "the new sizes are too short"); + Put64(p, roff + NAllocated, U64(data.Array, data.Offset)); + Put64(p, roff + NValidSize, U64(data.Array, data.Offset + 8)); + Put64(p, roff + NDataSize, U64(data.Array, data.Offset + 16)); + if (data.Count >= 32 && runsOffset >= NTotalSize + 8) + { + Put64(p, roff + NTotalSize, U64(data.Array, data.Offset + 24)); + } + + return; + + case NtfsLogOperation.AddIndexEntryRoot: + AddIndexEntryRoot(p, record, data); + return; + + case NtfsLogOperation.DeleteIndexEntryRoot: + DeleteIndexEntryRoot(p, record); + return; + + case NtfsLogOperation.SetIndexEntryVcnRoot: + { + var entry = RequireRootEntry(p, record, out _); + SetEntryVcn(p, entry, data, record); + return; + } + + case NtfsLogOperation.UpdateFileNameRoot: + { + var entry = RequireRootEntry(p, record, out _); + UpdateFileName(p, entry, data, record); + return; + } + + case NtfsLogOperation.UpdateRecordDataRoot: + { + var entry = RequireRootEntry(p, record, out _); + UpdateRecordData(p, entry, data, record); + return; + } + + case NtfsLogOperation.ZeroEndOfFileRecord: + Require(roff + data.Count <= size, record, "the zeroed range does not fit"); + Array.Clear(p, roff, data.Count); + return; + + default: + throw Unsupported(op, record); + } + } + + /// + /// Applies with to an index buffer. + /// + public static void ApplyToIndexBuffer(NtfsLogOperation op, NtfsLogRecord record, byte[] q, ArraySegment data) + { + var aoff = record.AttributeOffset; + switch (op) + { + case NtfsLogOperation.AddIndexEntryAllocation: + { + RequireIndexBufferEntry(q, aoff, record, allowEnd: true); + var used = (int)U32(q, IndxHeader + IhUsed); + var total = (int)U32(q, IndxHeader + IhTotal); + var length = EntryLength(data, record); + Require(used + length <= total && IndxHeader + used + length <= q.Length, record, "the index buffer is full"); + var end = IndxHeader + used; + Array.Copy(q, aoff, q, aoff + length, end - aoff); + Array.Copy(data.Array, data.Offset, q, aoff, length); + Put32(q, IndxHeader + IhUsed, (uint)(used + length)); + return; + } + + case NtfsLogOperation.DeleteIndexEntryAllocation: + { + RequireIndexBufferEntry(q, aoff, record); + var used = (int)U32(q, IndxHeader + IhUsed); + var end = IndxHeader + used; + var length = U16(q, aoff + IeLength); + Require(length > 0 && aoff + length <= end, record, "the index entry runs past the used part"); + Array.Copy(q, aoff + length, q, aoff, end - aoff - length); + Put32(q, IndxHeader + IhUsed, (uint)(used - length)); + return; + } + + case NtfsLogOperation.WriteEndOfIndexBuffer: + { + RequireIndexBufferEntry(q, aoff, record, allowEnd: true); + var total = (int)U32(q, IndxHeader + IhTotal); + Require(aoff + data.Count <= IndxHeader + total && aoff + data.Count <= q.Length, record, + "the end of the index buffer does not fit"); + Put32(q, IndxHeader + IhUsed, (uint)(aoff - IndxHeader + data.Count)); + Copy(data, q, aoff); + return; + } + + case NtfsLogOperation.SetIndexEntryVcnAllocation: + RequireIndexBufferEntry(q, aoff, record); + SetEntryVcn(q, aoff, data, record); + return; + + case NtfsLogOperation.UpdateFileNameAllocation: + RequireIndexBufferEntry(q, aoff, record); + UpdateFileName(q, aoff, data, record); + return; + + case NtfsLogOperation.UpdateRecordDataAllocation: + RequireIndexBufferEntry(q, aoff, record); + UpdateRecordData(q, aoff, data, record); + return; + + default: + throw Unsupported(op, record); + } + } + + /// Sets () or clears a range of bits. + public static void ChangeBits(byte[] bytes, int firstBit, int count, bool set) + { + for (var bit = firstBit; bit < firstBit + count; bit++) + { + if (set) + { + bytes[bit >> 3] |= (byte)(1 << (bit & 7)); + } + else + { + bytes[bit >> 3] &= (byte)~(1 << (bit & 7)); + } + } + } + + public static JournalException Unsupported(NtfsLogOperation op, NtfsLogRecord record) => + new($"Log record {record.Lsn:X} needs operation {op}, which DiscUtils cannot apply"); + + private static void CreateAttribute(byte[] p, NtfsLogRecord record, ArraySegment data) + { + var roff = record.RecordOffset; + RequireAttributeAt(p, roff, record, allowEnd: true); + Require(data.Count >= 0x18, record, "the new attribute is too short"); + var length = (int)U32(data.Array, data.Offset + ALength); + var used = (int)U32(p, FrBytesInUse); + Require(length >= 0x18 && (length & 7) == 0 && length <= data.Count, record, "the new attribute has a bad length"); + Require(used + length <= p.Length, record, "the new attribute does not fit in the file record"); + + Array.Copy(p, roff, p, roff + length, used - roff); + Array.Copy(data.Array, data.Offset, p, roff, length); + Put32(p, FrBytesInUse, (uint)(used + length)); + var id = U16(p, roff + AId); + if (U16(p, FrNextAttributeId) <= id) + { + Put16(p, FrNextAttributeId, (ushort)(id + 1)); + } + + if (IsIndexed(p, roff)) + { + Put16(p, FrHardLinks, (ushort)(U16(p, FrHardLinks) + 1)); + } + } + + private static void DeleteAttribute(byte[] p, NtfsLogRecord record) + { + var roff = record.RecordOffset; + RequireAttributeAt(p, roff, record); + var used = (int)U32(p, FrBytesInUse); + var length = (int)U32(p, roff + ALength); + Require(roff + length <= used, record, "the attribute runs past the used part of the file record"); + if (IsIndexed(p, roff)) + { + Put16(p, FrHardLinks, (ushort)(U16(p, FrHardLinks) - 1)); + } + + // Only the used part moves; the bytes left behind past the new end are unused (Windows logs a + // ZeroEndOfFileRecord when it wants them cleared). + Array.Copy(p, roff + length, p, roff, used - roff - length); + Put32(p, FrBytesInUse, (uint)(used - length)); + } + + private static void UpdateResidentValue(byte[] p, NtfsLogRecord record, ArraySegment data) + { + var roff = record.RecordOffset; + var aoff = record.AttributeOffset; + RequireAttributeAt(p, roff, record); + var length = (int)U32(p, roff + ALength); + if (record.RedoLength == record.UndoLength) + { + // Same size before and after: an in-place overwrite of bytes of the attribute record. Windows also + // uses it for headers, of non-resident attributes too (e.g. the compressed or sparse flag). + Require(aoff + data.Count <= length, record, "the value update runs past the attribute"); + Copy(data, p, roff + aoff); + return; + } + + // Otherwise a resident value is resized: it now ends where the data ends. + Require(p[roff + ANonResident] == 0, record, "a resident value update of a non-resident attribute"); + var valueOffset = U16(p, roff + RValueOffset); + Require(valueOffset >= 0x18 && valueOffset <= length && aoff >= valueOffset, record, + "the value update starts inside the attribute header or the value offset is invalid"); + var newLength = Align8(aoff + data.Count); + ResizeAttribute(p, roff, length, newLength, record); + Put32(p, roff + RValueLength, (uint)(aoff + data.Count - valueOffset)); + Copy(data, p, roff + aoff); + } + + private static void UpdateMappingPairs(byte[] p, NtfsLogRecord record, ArraySegment data) + { + var roff = record.RecordOffset; + var aoff = record.AttributeOffset; + RequireAttributeAt(p, roff, record); + Require(p[roff + ANonResident] != 0, record, "a run list update of a resident attribute"); + var length = (int)U32(p, roff + ALength); + Require(length >= NValidSize + 8, record, "the non-resident attribute header is too short"); + var runsOffset = U16(p, roff + NRunsOffset); + Require(runsOffset >= NValidSize + 8 && aoff >= runsOffset && aoff <= length, record, "the run list update is outside the run list"); + var newLength = Align8(aoff + data.Count); + Require(newLength > runsOffset, record, "the updated attribute has no mapping pairs"); + var vcn = (long)U64(p, roff + NStartVcn); + Require(vcn >= 0, record, "the run list has an invalid starting VCN"); + ResizeAttribute(p, roff, length, newLength, record); + Copy(data, p, roff + aoff); + + // The attribute's last VCN follows from its runs. + var position = roff + runsOffset; + var end = roff + newLength; + while (position < end && p[position] != 0) + { + var header = p[position]; + var lengthSize = header & 0x0F; + var offsetSize = header >> 4; + Require(lengthSize is > 0 and <= 8 && offsetSize <= 8 && position + 1 + lengthSize + offsetSize <= end, record, + "the new run list is damaged"); + long runLength = 0; + for (var i = lengthSize - 1; i >= 0; i--) + { + runLength = (runLength << 8) | p[position + 1 + i]; + } + + Require(runLength > 0 && runLength <= long.MaxValue - vcn, record, "the new run list has an invalid length or VCN overflow"); + vcn += runLength; + position += 1 + lengthSize + offsetSize; + } + + Put64(p, roff + NLastVcn, (ulong)(vcn - 1)); + } + + private static void AddIndexEntryRoot(byte[] p, NtfsLogRecord record, ArraySegment data) + { + var roff = record.RecordOffset; + var entry = RequireRootEntry(p, record, out var header, allowEnd: true); + var length = EntryLength(data, record); + var used = (int)U32(p, FrBytesInUse); + Require(used + length <= p.Length, record, "the index entry does not fit in the file record"); + + // Insert the entry, which moves the following entries and the following attributes up. + Array.Copy(p, entry, p, entry + length, used - entry); + Array.Copy(data.Array, data.Offset, p, entry, length); + Put32(p, FrBytesInUse, (uint)(used + length)); + Put32(p, roff + ALength, U32(p, roff + ALength) + (uint)length); + Put32(p, roff + RValueLength, U32(p, roff + RValueLength) + (uint)length); + Put32(p, header + IhUsed, U32(p, header + IhUsed) + (uint)length); + Put32(p, header + IhTotal, U32(p, header + IhTotal) + (uint)length); + } + + private static void DeleteIndexEntryRoot(byte[] p, NtfsLogRecord record) + { + var roff = record.RecordOffset; + var entry = RequireRootEntry(p, record, out var header); + var length = U16(p, entry + IeLength); + var entriesEnd = header + (int)U32(p, header + IhUsed); + Require(length > 0 && entry + length <= entriesEnd, record, "the index entry runs past the used part"); + var used = (int)U32(p, FrBytesInUse); + Array.Copy(p, entry + length, p, entry, used - entry - length); + Put32(p, FrBytesInUse, (uint)(used - length)); + Put32(p, roff + ALength, U32(p, roff + ALength) - length); + Put32(p, roff + RValueLength, U32(p, roff + RValueLength) - length); + Put32(p, header + IhUsed, U32(p, header + IhUsed) - length); + Put32(p, header + IhTotal, U32(p, header + IhTotal) - length); + } + + private static void SetEntryVcn(byte[] page, int entry, ArraySegment data, NtfsLogRecord record) + { + var length = U16(page, entry + IeLength); + Require(data.Count >= 8 && length >= 0x18 && entry + length <= page.Length, record, "the index entry has no child VCN"); + Array.Copy(data.Array, data.Offset, page, entry + length - 8, 8); + } + + private static void UpdateFileName(byte[] page, int entry, ArraySegment data, NtfsLogRecord record) + { + var length = U16(page, entry + IeLength); + var count = Math.Min(data.Count, DuplicatedInfoLength); + Require(length >= FileNameDuplicatedInfo + DuplicatedInfoLength && entry + length <= page.Length, record, + "the index entry has no file name"); + Array.Copy(data.Array, data.Offset, page, entry + FileNameDuplicatedInfo, count); + } + + private static void UpdateRecordData(byte[] page, int entry, ArraySegment data, NtfsLogRecord record) + { + var length = U16(page, entry + IeLength); + var dataOffset = U16(page, entry); + Require(dataOffset + data.Count <= length && entry + length <= page.Length, record, + "the index entry's data does not fit"); + Copy(data, page, entry + dataOffset); + } + + private static void ResizeAttribute(byte[] p, int roff, int oldLength, int newLength, NtfsLogRecord record) + { + if (newLength == oldLength) + { + return; + } + + // Like Windows, only the following attributes move; padding keeps whatever bytes were there. + var used = (int)U32(p, FrBytesInUse); + Require(used - oldLength + newLength <= p.Length, record, "the attribute does not fit in the file record"); + Array.Copy(p, roff + oldLength, p, roff + newLength, used - roff - oldLength); + Put32(p, roff + ALength, (uint)newLength); + Put32(p, FrBytesInUse, (uint)(used - oldLength + newLength)); + } + + /// Checks that an attribute (or, with , the end marker) starts at . + private static void RequireAttributeAt(byte[] p, int roff, NtfsLogRecord record, bool allowEnd = false) + { + var used = (int)U32(p, FrBytesInUse); + var position = (int)U16(p, FrFirstAttribute); + Require(used <= p.Length && position >= 0x20 && used >= position + 4, record, "the file record header is damaged"); + while (position < roff) + { + Require(position + 8 <= used, record, "the attribute list runs past the used part"); + if (U32(p, position + AType) == AttributeEnd) + { + break; + } + + var length = (int)U32(p, position + ALength); + Require(length >= 0x18 && length <= used - position, record, "an attribute has a bad length"); + position += length; + } + + Require(position == roff && roff + 4 <= used, record, $"no attribute starts at offset {roff:X} of the file record"); + if (U32(p, roff + AType) == AttributeEnd) + { + Require(allowEnd, record, $"offset {roff:X} is the end of the file record, not an attribute"); + return; + } + + Require(roff + 8 <= used, record, "the attribute header is truncated"); + var attributeLength = U32(p, roff + ALength); + Require(attributeLength >= 0x18 && attributeLength <= used - roff, record, "the attribute has a bad length"); + } + + /// + /// Checks that an index entry of the INDEX_ROOT at the logged attribute offset starts at the logged entry + /// offset; returns the entry's offset in the record and the index header's. + /// + private static int RequireRootEntry(byte[] p, NtfsLogRecord record, out int header, bool allowEnd = false) + { + var roff = record.RecordOffset; + RequireAttributeAt(p, roff, record); + Require(U32(p, roff + AType) == AttributeIndexRoot && p[roff + ANonResident] == 0, record, + "the attribute is not an index root"); + var attributeLength = (int)U32(p, roff + ALength); + var valueOffset = U16(p, roff + RValueOffset); + var valueLength = U32(p, roff + RValueLength); + Require(valueOffset >= 0x18 && valueLength >= 0x20 && valueLength <= attributeLength - valueOffset, + record, "the index root value is outside the attribute"); + header = roff + valueOffset + 0x10; + var target = roff + record.AttributeOffset; + RequireIndexEntry(p, header, (int)valueLength - 0x10, target, record, allowEnd); + return target; + } + + private static void RequireIndexBufferEntry(byte[] q, int aoff, NtfsLogRecord record, bool allowEnd = false) => + RequireIndexEntry(q, IndxHeader, q.Length - IndxHeader, aoff, record, allowEnd); + + private static void RequireIndexEntry(byte[] page, int header, int capacity, int target, NtfsLogRecord record, bool allowEnd) + { + Require(capacity >= 0x10, record, "the index header is truncated"); + var entries = U32(page, header + IhEntriesOffset); + var used = U32(page, header + IhUsed); + var total = U32(page, header + IhTotal); + Require(entries >= 0x10 && entries <= used && used <= total && total <= capacity, + record, "the index header is damaged"); + var position = header + (int)entries; + var end = header + (int)used; + while (position < target && position < end) + { + position += CheckedEntryLength(page, position, end, record); + } + + Require(position == target && target <= end, record, $"no index entry starts at offset {target:X}"); + if (!allowEnd) + { + CheckedEntryLength(page, target, end, record); + } + } + + private static int CheckedEntryLength(byte[] page, int offset, int end, NtfsLogRecord record) + { + Require(offset <= end - 0x10, record, "an index entry header is truncated"); + var length = U16(page, offset + IeLength); + Require(length >= 0x10 && length <= end - offset, record, "an index entry has a bad length"); + return length; + } + + private static int EntryLength(ArraySegment data, NtfsLogRecord record) + { + Require(data.Count >= 0x10, record, "the index entry is too short"); + int length = U16(data.Array, data.Offset + IeLength); + Require(length >= 0x10 && length <= data.Count && (length & 7) == 0, record, "the index entry has a bad length"); + return length; + } + + private static bool IsIndexed(byte[] p, int roff) => p[roff + ANonResident] == 0 && (p[roff + RIndexed] & 1) != 0; + + private static void Copy(ArraySegment data, byte[] target, int offset) => + Array.Copy(data.Array, data.Offset, target, offset, data.Count); + + private static void Require(bool condition, NtfsLogRecord record, string what) + { + if (!condition) + { + throw new JournalException($"Log record {record.Lsn:X} ({record.RedoOperation}/{record.UndoOperation}) does not fit the volume: {what}"); + } + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsLogAnalysis.cs b/Library/DiscUtils.Ntfs/Journal/NtfsLogAnalysis.cs new file mode 100644 index 000000000..722d2803e --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsLogAnalysis.cs @@ -0,0 +1,435 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// The analysis pass of NTFS recovery: starting from the tables of the last checkpoint, it reads the active NTFS +/// client's records after the checkpoint and works out which pages may be stale on disk (the dirty page table) +/// and which transactions were not finished (the transaction table). +/// +internal sealed class NtfsLogAnalysis +{ + private NtfsLogAnalysis(LfsLog log, NtfsRestartRecord restart) + { + Log = log; + RestartRecord = restart; + } + + public LfsLog Log { get; } + + public NtfsRestartRecord RestartRecord { get; } + + /// The open attribute table, keyed by entry offset (what log records call the target attribute). + public Dictionary OpenAttributes { get; } = []; + + public List DirtyPages { get; } = []; + + /// The transaction table, keyed by entry offset (the transaction id of log records). + public Dictionary Transactions { get; } = []; + + /// Clusters per dirty page (from the checkpoint's table, else one log page worth). + public int ClustersPerPage { get; private set; } + + /// Where the redo pass starts. + public ulong RedoLsn { get; private set; } + + /// The last record of the log. + public ulong LastLsn { get; private set; } + + public int RecordsAfterCheckpoint { get; private set; } + + /// Transactions that were still running at the crash and have changes to roll back. + public IEnumerable> Losers => + Transactions.Where(t => t.Value.State == TransactionState.Active && t.Value.UndoNextLsn != 0); + + /// Analyzes the active NTFS client's log from its last checkpoint. + public static NtfsLogAnalysis Run(LfsLog log, JournalVolumeLayout layout) + { + var client = log.NtfsClient; + var restartLfs = log.ReadRecord(client.RestartLsn) + ?? throw new JournalException($"The NTFS restart record at {client.RestartLsn:X} is not in the log"); + log.RequireNtfsRecord(restartLfs, "NTFS restart record"); + var analysis = new NtfsLogAnalysis(log, NtfsRestartRecord.Parse(restartLfs)); + analysis.LoadCheckpointTables(layout); + analysis.ReadAfterCheckpoint(); + return analysis; + } + + public DirtyPageEntry FindDirtyPage(uint targetAttribute, long vcn) + { + foreach (var page in DirtyPages) + { + if (page.Covers(targetAttribute, vcn)) + { + return page; + } + } + + return null; + } + + /// + /// The oldest LSN from which changes to the page at (, ) + /// may be missing. With clusters bigger than a page (64K clusters, 4K pages) several entries share one VCN + /// without saying which page of the cluster they are; the oldest of them counts. + /// + public ulong OldestLsnFor(uint targetAttribute, long vcn) + { + ulong oldest = ulong.MaxValue; + foreach (var page in DirtyPages) + { + if (page.Covers(targetAttribute, vcn) && page.OldestLsn < oldest) + { + oldest = page.OldestLsn; + } + } + + return oldest; + } + + private byte[] ReadTableDump(ulong lsn, int length, string name) + { + if (length == 0) + { + return null; + } + + var lfs = Log.ReadRecord(lsn) ?? throw new JournalException($"The {name} dump at {lsn:X} is not in the log"); + Log.RequireNtfsRecord(lfs, $"{name} dump", LfsRecord.TypeClientRecord); + var record = NtfsLogRecord.TryParse(lfs, out var error) ?? throw new JournalException(error); + var data = record.RedoData; + if (data.Count < length) + { + throw new JournalException($"The {name} dump at {lsn:X} is shorter than the restart record says"); + } + + return data.ToArray(); + } + + private void LoadCheckpointTables(JournalVolumeLayout layout) + { + var r = RestartRecord; + var version = r.MajorVersion; + + var openAttributes = ReadTableDump(r.OpenAttributeTableLsn, r.OpenAttributeTableLength, "open attribute table"); + if (openAttributes != null) + { + foreach (var (offset, size) in RestartTable.Entries(openAttributes)) + { + var expected = version == 0 ? OpenAttributeEntry.SizeVersion0 : OpenAttributeEntry.SizeVersion1; + if (size != expected) + { + throw new JournalException($"Open attribute table entries are {size} bytes, expected {expected}"); + } + + OpenAttributes[offset] = OpenAttributeEntry.Parse(openAttributes, offset, version); + } + } + + var names = ReadTableDump(r.AttributeNamesLsn, r.AttributeNamesLength, "attribute names"); + if (names != null) + { + foreach (var (offset, name) in RestartTable.AttributeNames(names)) + { + if (OpenAttributes.TryGetValue(offset, out var entry)) + { + entry.Name = name; + } + } + } + + var dirtyPages = ReadTableDump(r.DirtyPageTableLsn, r.DirtyPageTableLength, "dirty page table"); + var lcnsOffset = version == 0 ? 0x24 : 0x20; + ClustersPerPage = Math.Max(1, Log.PageSize / layout.BytesPerCluster); + if (dirtyPages != null) + { + var first = true; + foreach (var (offset, size) in RestartTable.Entries(dirtyPages)) + { + if (first) + { + ClustersPerPage = (size - lcnsOffset) / 8; + first = false; + } + + var page = DirtyPageEntry.Parse(dirtyPages, offset, size, version); + CheckHistoryRecord(page.OldestLsn, "dirty page's oldest record"); + DirtyPages.Add(page); + } + + if (first && dirtyPages.Length >= 2) + { + ClustersPerPage = Math.Max(1, (U16(dirtyPages, 0) - lcnsOffset) / 8); + } + } + + var transactions = ReadTableDump(r.TransactionTableLsn, r.TransactionTableLength, "transaction table"); + if (transactions != null) + { + foreach (var (offset, size) in RestartTable.Entries(transactions)) + { + if (size < TransactionEntry.Size) + { + throw new JournalException("A transaction table entry is too short"); + } + + var transaction = TransactionEntry.Parse(transactions, offset); + if (transaction.State is not (TransactionState.Active or TransactionState.Prepared or TransactionState.Committed)) + { + throw new JournalException($"Transaction {offset:X} has an invalid checkpoint state ({transaction.State})"); + } + + CheckHistoryRecord(transaction.FirstLsn, "transaction's first record"); + CheckHistoryRecord(transaction.PreviousLsn, "transaction's previous record"); + CheckHistoryRecord(transaction.UndoNextLsn, "transaction's next undo record"); + Transactions[(uint)offset] = transaction; + } + } + + if (ClustersPerPage < 1 || (ClustersPerPage & (ClustersPerPage - 1)) != 0) + { + throw new JournalException($"A dirty page covers {ClustersPerPage} clusters, which is not a power of two"); + } + } + + private void CheckHistoryRecord(ulong lsn, string role) + { + // Check retained history here. Redo and undo still decide which records must be present to recover. + if (Log.ReadRecord(lsn, readClientData: false) is { } record) + { + Log.RequireNtfsRecord(record, role, LfsRecord.TypeClientRecord); + } + } + + private void ReadAfterCheckpoint() + { + var checkpoint = RestartRecord.CheckpointLsn; + ulong firstAfterCheckpoint = 0; + foreach (var lfs in Log.ReadForward(checkpoint)) + { + // The end of the shared LFS log is used for the restart pages and as an upper bound for page LSNs. + LastLsn = lfs.Lsn; + var afterCheckpoint = lfs.Lsn != checkpoint; + if (afterCheckpoint) + { + RecordsAfterCheckpoint++; + } + else + { + Log.RequireNtfsRecord(lfs, "checkpoint record"); + } + + if (!Log.IsForNtfsClient(lfs)) + { + continue; + } + + if (afterCheckpoint && firstAfterCheckpoint == 0) + { + firstAfterCheckpoint = lfs.Lsn; + } + + if (lfs.RecordType != LfsRecord.TypeClientRecord) + { + continue; + } + + var record = NtfsLogRecord.TryParse(lfs, out var error) ?? throw new JournalException(error); + if (afterCheckpoint) + { + Analyze(record); + } + } + + if (LastLsn == 0) + { + throw new JournalException($"The checkpoint record {checkpoint:X} is not in the log"); + } + + var redo = firstAfterCheckpoint; + foreach (var page in DirtyPages) + { + if (page.OldestLsn != 0 && (redo == 0 || page.OldestLsn < redo)) + { + redo = page.OldestLsn; + } + } + + foreach (var entry in Transactions) + { + var transaction = entry.Value; + if (transaction.State == TransactionState.Prepared) + { + throw new JournalException($"Prepared transaction {entry.Key:X} has no recorded outcome"); + } + + if (transaction.FirstLsn != 0 && (redo == 0 || transaction.FirstLsn < redo)) + { + redo = transaction.FirstLsn; + } + } + + RedoLsn = redo; + } + + private void Analyze(NtfsLogRecord record) + { + var lfs = record.Lfs; + if (lfs.TransactionId == 0) + { + throw new JournalException($"Log record {lfs.Lsn:X} has no transaction"); + } + + if (!Transactions.TryGetValue(lfs.TransactionId, out var transaction)) + { + transaction = new TransactionEntry { State = TransactionState.Active, FirstLsn = lfs.Lsn }; + Transactions[lfs.TransactionId] = transaction; + } + + transaction.PreviousLsn = lfs.Lsn; + transaction.UndoNextLsn = record.UndoOperation == NtfsLogOperation.CompensationLogRecord ? lfs.ClientUndoNextLsn : lfs.Lsn; + + switch (record.RedoOperation) + { + case NtfsLogOperation.DeleteDirtyClusters: + DeleteDirtyClusters(record); + return; + + case NtfsLogOperation.OpenNonresidentAttribute: + OpenAttribute(record); + return; + + case NtfsLogOperation.HotFix: + var hot = FindDirtyPage(record.TargetAttribute, record.TargetVcn); + if (hot != null && record.Lcns.Length > 0 && hot.Lcns[record.TargetVcn - hot.Vcn] != 0) + { + hot.Lcns[record.TargetVcn - hot.Vcn] = record.Lcns[0]; + } + + return; + + case NtfsLogOperation.EndTopLevelAction: + transaction.PreviousLsn = lfs.Lsn; + transaction.UndoNextLsn = lfs.ClientUndoNextLsn; + return; + + case NtfsLogOperation.PrepareTransaction: + transaction.State = TransactionState.Prepared; + return; + + case NtfsLogOperation.CommitTransaction: + transaction.State = TransactionState.Committed; + return; + + case NtfsLogOperation.ForgetTransaction: + Transactions.Remove(lfs.TransactionId); + return; + } + + var target = NtfsLogOperations.TargetOf(record.RedoOperation); + if (target == NtfsLogTarget.None || record.Lcns.Length == 0) + { + return; + } + + if (record.TargetVcn < 0 || record.TargetVcn > long.MaxValue - ClustersPerPage) + { + throw new JournalException($"Log record {record.Lsn:X} has an invalid target VCN"); + } + + // Every record that changes a page marks that page dirty from this LSN on, with the clusters it names. + var page = FindDirtyPage(record.TargetAttribute, record.TargetVcn); + if (page == null) + { + page = new DirtyPageEntry + { + TargetAttribute = record.TargetAttribute, + Vcn = record.TargetVcn & ~((long)ClustersPerPage - 1), + OldestLsn = lfs.Lsn, + Lcns = new long[ClustersPerPage], + }; + DirtyPages.Add(page); + } + + var first = record.TargetVcn - page.Vcn; + if (first + record.Lcns.Length > page.Lcns.Length) + { + throw new JournalException($"Log record {lfs.Lsn:X} names clusters beyond its dirty page"); + } + + Array.Copy(record.Lcns, 0, page.Lcns, first, record.Lcns.Length); + } + + private void DeleteDirtyClusters(NtfsLogRecord record) + { + var data = record.RedoData; + for (var i = 0; i + 16 <= data.Count; i += 16) + { + var lcn = (long)U64(data.Array, data.Offset + i); + var length = (long)U64(data.Array, data.Offset + i + 8); + if (lcn < 0 || length < 0 || length > long.MaxValue - lcn) + { + throw new JournalException($"Log record {record.Lsn:X} has an invalid range of freed clusters"); + } + + foreach (var page in DirtyPages) + { + for (var j = 0; j < page.Lcns.Length; j++) + { + if (page.Lcns[j] >= lcn && page.Lcns[j] < lcn + length) + { + page.Lcns[j] = 0; + } + } + } + } + } + + private void OpenAttribute(NtfsLogRecord record) + { + var data = record.RedoData; + var version = RestartRecord.MajorVersion; + var size = version == 0 ? OpenAttributeEntry.SizeVersion0 : OpenAttributeEntry.SizeVersion1; + if (data.Count < size - 8) + { + throw new JournalException($"Log record {record.Lsn:X} opens an attribute with too little data"); + } + + var buffer = new byte[size]; + Array.Copy(data.Array, data.Offset, buffer, 0, Math.Min(size, data.Count)); + var entry = OpenAttributeEntry.Parse(buffer, 0, version); + var name = record.UndoData; + if (name.Count > 0) + { + entry.Name = Encoding.Unicode.GetString(name.Array, name.Offset, name.Count); + } + + OpenAttributes[record.TargetAttribute] = entry; + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsLogOperation.cs b/Library/DiscUtils.Ntfs/Journal/NtfsLogOperation.cs new file mode 100644 index 000000000..2ff9c432f --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsLogOperation.cs @@ -0,0 +1,148 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +namespace DiscUtils.Ntfs.Journal; + +/// +/// The operations NTFS writes into its log, as redo and undo actions of a log record. +/// +internal enum NtfsLogOperation : ushort +{ + Noop = 0x00, + CompensationLogRecord = 0x01, + InitializeFileRecordSegment = 0x02, + DeallocateFileRecordSegment = 0x03, + WriteEndOfFileRecordSegment = 0x04, + CreateAttribute = 0x05, + DeleteAttribute = 0x06, + UpdateResidentValue = 0x07, + UpdateNonresidentValue = 0x08, + UpdateMappingPairs = 0x09, + DeleteDirtyClusters = 0x0A, + SetNewAttributeSizes = 0x0B, + AddIndexEntryRoot = 0x0C, + DeleteIndexEntryRoot = 0x0D, + AddIndexEntryAllocation = 0x0E, + DeleteIndexEntryAllocation = 0x0F, + WriteEndOfIndexBuffer = 0x10, + SetIndexEntryVcnRoot = 0x11, + SetIndexEntryVcnAllocation = 0x12, + UpdateFileNameRoot = 0x13, + UpdateFileNameAllocation = 0x14, + SetBitsInNonresidentBitMap = 0x15, + ClearBitsInNonresidentBitMap = 0x16, + HotFix = 0x17, + EndTopLevelAction = 0x18, + PrepareTransaction = 0x19, + CommitTransaction = 0x1A, + ForgetTransaction = 0x1B, + OpenNonresidentAttribute = 0x1C, + OpenAttributeTableDump = 0x1D, + AttributeNamesDump = 0x1E, + DirtyPageTableDump = 0x1F, + TransactionTableDump = 0x20, + UpdateRecordDataRoot = 0x21, + UpdateRecordDataAllocation = 0x22, + UpdateRelativeDataInIndex = 0x23, + UpdateRelativeDataInIndex2 = 0x24, + ZeroEndOfFileRecord = 0x25, +} + +/// +/// What kind of on-disk structure an operation changes. +/// +internal enum NtfsLogTarget +{ + /// The operation changes nothing on disk (bookkeeping, transaction state, table dumps). + None, + + /// A file record of the master file table. + FileRecord, + + /// An index buffer (INDX) of an $INDEX_ALLOCATION attribute. + IndexBuffer, + + /// The bytes of a non-resident attribute (bitmaps, or any attribute for UpdateNonresidentValue). + NonResident, + + /// An operation whose meaning DiscUtils does not know. + Unknown, +} + +internal static class NtfsLogOperations +{ + /// + /// Gets which structure an operation changes when it is applied as a redo or undo action. + /// + public static NtfsLogTarget TargetOf(NtfsLogOperation op) + { + switch (op) + { + case NtfsLogOperation.Noop: + case NtfsLogOperation.CompensationLogRecord: + case NtfsLogOperation.DeleteDirtyClusters: + case NtfsLogOperation.HotFix: + case NtfsLogOperation.EndTopLevelAction: + case NtfsLogOperation.PrepareTransaction: + case NtfsLogOperation.CommitTransaction: + case NtfsLogOperation.ForgetTransaction: + case NtfsLogOperation.OpenNonresidentAttribute: + case NtfsLogOperation.OpenAttributeTableDump: + case NtfsLogOperation.AttributeNamesDump: + case NtfsLogOperation.DirtyPageTableDump: + case NtfsLogOperation.TransactionTableDump: + return NtfsLogTarget.None; + + case NtfsLogOperation.InitializeFileRecordSegment: + case NtfsLogOperation.DeallocateFileRecordSegment: + case NtfsLogOperation.WriteEndOfFileRecordSegment: + case NtfsLogOperation.CreateAttribute: + case NtfsLogOperation.DeleteAttribute: + case NtfsLogOperation.UpdateResidentValue: + case NtfsLogOperation.UpdateMappingPairs: + case NtfsLogOperation.SetNewAttributeSizes: + case NtfsLogOperation.AddIndexEntryRoot: + case NtfsLogOperation.DeleteIndexEntryRoot: + case NtfsLogOperation.SetIndexEntryVcnRoot: + case NtfsLogOperation.UpdateFileNameRoot: + case NtfsLogOperation.UpdateRecordDataRoot: + case NtfsLogOperation.ZeroEndOfFileRecord: + return NtfsLogTarget.FileRecord; + + case NtfsLogOperation.AddIndexEntryAllocation: + case NtfsLogOperation.DeleteIndexEntryAllocation: + case NtfsLogOperation.WriteEndOfIndexBuffer: + case NtfsLogOperation.SetIndexEntryVcnAllocation: + case NtfsLogOperation.UpdateFileNameAllocation: + case NtfsLogOperation.UpdateRecordDataAllocation: + return NtfsLogTarget.IndexBuffer; + + case NtfsLogOperation.UpdateNonresidentValue: + case NtfsLogOperation.SetBitsInNonresidentBitMap: + case NtfsLogOperation.ClearBitsInNonresidentBitMap: + return NtfsLogTarget.NonResident; + + default: + return NtfsLogTarget.Unknown; + } + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsLogPages.cs b/Library/DiscUtils.Ntfs/Journal/NtfsLogPages.cs new file mode 100644 index 000000000..d698e45fb --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsLogPages.cs @@ -0,0 +1,380 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +internal enum PageKind +{ + FileRecord, + IndexBuffer, + Raw, +} + +/// +/// A file record or index buffer being replayed, in its logical form (update sequence array removed). +/// +internal sealed class LogicalPage +{ + public PageKind Kind { get; set; } + + /// Where every byte of the page lives: the page's clusters and the offset of the page in the first. + public long[] Lcns { get; set; } + + public int StartInFirstCluster { get; set; } + + public byte[] Data { get; set; } + + /// The signature and update sequence array are intact (or the replay made them so). + public bool Valid { get; set; } + + public bool Dirty { get; set; } + + public uint Signature => U32(Data, 0); + + public ulong Lsn + { + get => U64(Data, 0x08); + set => Put64(Data, 0x08, value); + } +} + +/// +/// All volume pages the replay reads and changes. Nothing is written until , so a refused +/// replay leaves the volume untouched, and the same set of changes serves a write to disk or a read-only overlay. +/// +internal sealed class NtfsLogPages +{ + private readonly Stream _volume; + private readonly JournalVolumeLayout _layout; + private readonly Dictionary _logical = []; + private readonly Dictionary _rawClusters = []; + private readonly Dictionary _rawOriginal = []; + private readonly Dictionary _clusterUse = []; + private readonly Dictionary _undoOwners = []; + private readonly Dictionary> _undoBitOwners = []; + + public NtfsLogPages(Stream volume, JournalVolumeLayout layout) + { + _volume = volume; + _layout = layout; + } + + public int ClusterSize => _layout.BytesPerCluster; + + /// File records, index buffers and plain clusters whose bytes the replay changes (set by ). + public int ChangedPageCount { get; private set; } + + public int UndoTransactionCount => + _undoOwners.Values.Concat(_undoBitOwners.Values.SelectMany(runs => runs.Select(run => run.Transaction))).Distinct().Count(); + + /// + /// Gets a file record or index buffer of bytes that starts + /// bytes into the clusters . + /// + public LogicalPage GetLogical(PageKind kind, long[] lcns, int start, int length, uint undoTransaction) + { + var (spanLcns, startInFirst) = Span(lcns, start, length); + Claim(spanLcns, kind, undoTransaction); + var key = (spanLcns[0] * ClusterSize) + startInFirst; + if (_logical.TryGetValue(key, out var page)) + { + if (page.Kind != kind || page.Data.Length != length) + { + throw new JournalException($"The log uses the bytes at {key:X} both as {page.Kind} and as {kind}"); + } + + if (!page.Lcns.AsSpan().SequenceEqual(spanLcns)) + { + throw new JournalException($"The log gives conflicting cluster mappings for the page at {key:X}"); + } + + return page; + } + + var data = new byte[length]; + ReadSpan(spanLcns, startInFirst, data, 0, length); + var expected = kind == PageKind.FileRecord ? SignatureFile : SignatureIndx; + page = new LogicalPage + { + Kind = kind, + Lcns = spanLcns, + StartInFirstCluster = startInFirst, + Data = data, + }; + page.Valid = page.Signature == expected && RemoveFixups(data, 0, length); + _logical[key] = page; + return page; + } + + /// + /// Reads bytes of a non-resident attribute page that has no update sequence array (bitmaps, plain data). + /// + public byte[] ReadRaw(long[] lcns, int start, int length, uint undoTransaction) + { + var (spanLcns, startInFirst) = Span(lcns, start, length); + Claim(spanLcns, PageKind.Raw, undoTransaction); + var buffer = new byte[length]; + ReadSpan(spanLcns, startInFirst, buffer, 0, length); + return buffer; + } + + /// + /// Reads the bytes that hold bits of a bitmap, starting at bit + /// of the byte bytes into the clusters + /// . An undo claims only those bits, not the whole clusters: transactions that roll back + /// different bits of one cluster (two allocations in the same part of $Bitmap) do not conflict. + /// + public byte[] ReadBits(long[] lcns, int start, int firstBitInByte, int bits, uint undoTransaction) + { + var length = (int)((firstBitInByte + (long)bits + 7) >> 3); + var (spanLcns, startInFirst) = Span(lcns, start, length); + Claim(spanLcns, PageKind.Raw); + if (undoTransaction != 0) + { + var clusterBits = (long)ClusterSize * 8; + var first = ((long)startInFirst * 8) + firstBitInByte; + var end = first + bits; + for (var i = 0; i < spanLcns.Length; i++) + { + var from = Math.Max(first, i * clusterBits) - (i * clusterBits); + var to = Math.Min(end, (i + 1) * clusterBits) - (i * clusterBits); + ClaimBits(spanLcns[i], (int)from, (int)(to - from), undoTransaction); + } + } + + var buffer = new byte[length]; + ReadSpan(spanLcns, startInFirst, buffer, 0, length); + return buffer; + } + + public void WriteRaw(long[] lcns, int start, byte[] buffer, int offset, int length) + { + var (spanLcns, startInFirst) = Span(lcns, start, length); + Claim(spanLcns, PageKind.Raw); + var position = startInFirst; + foreach (var lcn in spanLcns) + { + // The span holds exactly the clusters the range touches. + var cluster = RawCluster(lcn); + var chunk = Math.Min(length, ClusterSize - position); + Array.Copy(buffer, offset, cluster, position, chunk); + offset += chunk; + length -= chunk; + position = 0; + } + } + + /// + /// Turns every change into whole clusters: logical pages get a fresh update sequence array. + /// + public Dictionary Commit() + { + // Plain clusters are replayed unconditionally; only those whose bytes really changed are written. + var clusters = new Dictionary(); + var changed = 0; + foreach (var pair in _rawClusters) + { + if (!pair.Value.AsSpan().SequenceEqual(_rawOriginal[pair.Key])) + { + clusters[pair.Key] = pair.Value; + changed++; + } + } + + foreach (var page in _logical.Values) + { + if (!page.Dirty) + { + continue; + } + + var bytes = (byte[])page.Data.Clone(); + var replaced = new byte[bytes.Length]; + ReadSpan(page.Lcns, page.StartInFirstCluster, replaced, 0, replaced.Length); + if (page.Valid) + { + ApplyFixups(bytes, 0, bytes.Length, replaced); + } + else if (bytes.AsSpan().SequenceEqual(replaced)) + { + continue; + } + + changed++; + var position = page.StartInFirstCluster; + var offset = 0; + foreach (var lcn in page.Lcns) + { + if (!clusters.TryGetValue(lcn, out var cluster)) + { + cluster = new byte[ClusterSize]; + ReadExactly(_volume, lcn * ClusterSize, cluster, 0, ClusterSize); + clusters[lcn] = cluster; + } + + var chunk = Math.Min(bytes.Length - offset, ClusterSize - position); + Array.Copy(bytes, offset, cluster, position, chunk); + offset += chunk; + position = 0; + if (offset == bytes.Length) + { + break; + } + } + } + + ChangedPageCount = changed; + return clusters; + } + + private (long[] Lcns, int StartInFirst) Span(long[] lcns, int start, int length) + { + // The callers checked that the range lies within the clusters (and that its first one was not freed). + var firstIndex = start / ClusterSize; + var lastIndex = (int)((start + (long)length - 1) / ClusterSize); + var span = new long[lastIndex - firstIndex + 1]; + var seen = span.Length > 1 ? new HashSet() : null; + for (var i = 0; i < span.Length; i++) + { + var lcn = lcns[firstIndex + i]; + if (!_layout.ContainsClusters(lcn, 1)) + { + throw new JournalException($"The log names cluster {lcn}, which is not on the volume"); + } + + if (seen != null && !seen.Add(lcn)) + { + throw new JournalException($"The log maps more than one part of a page to cluster {lcn}"); + } + + span[i] = lcn; + } + + return (span, start % ClusterSize); + } + + private void Claim(long[] lcns, PageKind kind, uint undoTransaction = 0) + { + foreach (var lcn in lcns) + { + if (_clusterUse.TryGetValue(lcn, out var existing) && existing != kind) + { + throw new JournalException($"The log uses cluster {lcn} both as {existing} and as {kind}"); + } + + _clusterUse[lcn] = kind; + if (undoTransaction != 0) + { + if (_undoOwners.TryGetValue(lcn, out var owner) && owner != undoTransaction) + { + throw new JournalException($"Undo for transactions {owner:X} and {undoTransaction:X} overlaps cluster {lcn}"); + } + + if (_undoBitOwners.TryGetValue(lcn, out var runs)) + { + foreach (var run in runs) + { + if (run.Transaction != undoTransaction) + { + throw new JournalException($"Undo for transactions {run.Transaction:X} and {undoTransaction:X} overlaps cluster {lcn}"); + } + } + } + + _undoOwners[lcn] = undoTransaction; + } + } + } + + /// + /// Claims bits to + - 1 of cluster + /// for the undo of one transaction. + /// + private void ClaimBits(long lcn, int first, int count, uint undoTransaction) + { + if (_undoOwners.TryGetValue(lcn, out var owner) && owner != undoTransaction) + { + throw new JournalException($"Undo for transactions {owner:X} and {undoTransaction:X} overlaps cluster {lcn}"); + } + + if (!_undoBitOwners.TryGetValue(lcn, out var runs)) + { + runs = []; + _undoBitOwners[lcn] = runs; + } + + foreach (var run in runs) + { + if (run.Transaction != undoTransaction && run.First < first + count && first < run.First + run.Count) + { + throw new JournalException( + $"Undo for transactions {run.Transaction:X} and {undoTransaction:X} overlaps bits {Math.Max(first, run.First)}-{Math.Min(first + count, run.First + run.Count) - 1} of cluster {lcn}"); + } + } + + runs.Add((first, count, undoTransaction)); + } + + private byte[] RawCluster(long lcn) + { + if (!_rawClusters.TryGetValue(lcn, out var cluster)) + { + cluster = new byte[ClusterSize]; + ReadExactly(_volume, lcn * ClusterSize, cluster, 0, ClusterSize); + _rawClusters[lcn] = cluster; + _rawOriginal[lcn] = (byte[])cluster.Clone(); + } + + return cluster; + } + + private void ReadSpan(long[] lcns, int startInFirst, byte[] buffer, int offset, int length) + { + var position = startInFirst; + foreach (var lcn in lcns) + { + var chunk = Math.Min(length, ClusterSize - position); + if (_rawClusters.TryGetValue(lcn, out var cached)) + { + Array.Copy(cached, position, buffer, offset, chunk); + } + else + { + ReadExactly(_volume, (lcn * ClusterSize) + position, buffer, offset, chunk); + } + + offset += chunk; + length -= chunk; + position = 0; + if (length == 0) + { + break; + } + } + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsLogRecord.cs b/Library/DiscUtils.Ntfs/Journal/NtfsLogRecord.cs new file mode 100644 index 000000000..2ed8ea234 --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsLogRecord.cs @@ -0,0 +1,437 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.Text; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// An NTFS log record: the redo and undo action of one step of a transaction, with the place it applies to. +/// +internal sealed class NtfsLogRecord +{ + public const int HeaderLength = 0x20; + + public LfsRecord Lfs { get; private set; } + + public ulong Lsn => Lfs.Lsn; + + public NtfsLogOperation RedoOperation { get; private set; } + + public NtfsLogOperation UndoOperation { get; private set; } + + public int RedoOffset { get; private set; } + + public int RedoLength { get; private set; } + + public int UndoOffset { get; private set; } + + public int UndoLength { get; private set; } + + /// The byte offset of the target attribute's entry in the open attribute table. + public ushort TargetAttribute { get; private set; } + + /// The offset of the attribute in a file record, or of the index buffer in a page. + public int RecordOffset { get; private set; } + + /// The offset inside the attribute (or of the index entry inside the index buffer). + public int AttributeOffset { get; private set; } + + /// Where the target starts inside the cluster at , in 512-byte units. + public int ClusterBlockOffset { get; private set; } + + public long TargetVcn { get; private set; } + + /// The clusters of the target page when the record was written, one per VCN from . + public long[] Lcns { get; private set; } + + /// LFS header flag: the redo data is all zeros and not stored (its place may hold the undo data). + public const ushort FlagRedoDataZero = 0x0002; + + /// LFS header flag: the undo data is all zeros and not stored. + public const ushort FlagUndoDataZero = 0x0004; + + /// + /// The redo data. Windows does not store data that is all zeros: flag 0x2 in the LFS header marks zero redo + /// data (seen when a resident value is zeroed, e.g. by making a range of a file sparse; the bytes at the redo + /// offset are then the undo data, stored once), flag 0x4 zero undo data. A record can also end before the + /// declared data does; the missing bytes count as zeros as well. + /// + public ArraySegment RedoData => Data(RedoOffset, RedoLength, (Lfs.Flags & FlagRedoDataZero) != 0); + + /// The undo data (see ). + public ArraySegment UndoData => Data(UndoOffset, UndoLength, (Lfs.Flags & FlagUndoDataZero) != 0); + + private ArraySegment Data(int offset, int length, bool zero) + { + var bytes = Lfs.ClientData; + if (length == 0) + { + return new ArraySegment(bytes, 0, 0); + } + + if (zero) + { + return new ArraySegment(new byte[length]); + } + + if (offset + length <= bytes.Length) + { + return new ArraySegment(bytes, offset, length); + } + + var padded = new byte[length]; + if (offset < bytes.Length) + { + Array.Copy(bytes, offset, padded, 0, bytes.Length - offset); + } + + return new ArraySegment(padded); + } + + /// + /// Parses the client data of an LFS client record; null and an error if it is not a well-formed NTFS + /// log record. + /// + public static NtfsLogRecord TryParse(LfsRecord lfs, out string error) + { + var data = lfs.ClientData; + error = null; + if (data.Length < HeaderLength) + { + error = $"log record {lfs.Lsn:X} is too short"; + return null; + } + + var record = new NtfsLogRecord + { + Lfs = lfs, + RedoOperation = (NtfsLogOperation)U16(data, 0x00), + UndoOperation = (NtfsLogOperation)U16(data, 0x02), + RedoOffset = U16(data, 0x04), + RedoLength = U16(data, 0x06), + UndoOffset = U16(data, 0x08), + UndoLength = U16(data, 0x0A), + TargetAttribute = U16(data, 0x0C), + RecordOffset = U16(data, 0x10), + AttributeOffset = U16(data, 0x12), + ClusterBlockOffset = U16(data, 0x14), + TargetVcn = (long)U64(data, 0x18), + }; + + if (NtfsLogOperations.TargetOf(record.RedoOperation) == NtfsLogTarget.Unknown || + NtfsLogOperations.TargetOf(record.UndoOperation) == NtfsLogTarget.Unknown) + { + error = $"Log record {lfs.Lsn:X} has an unsupported redo or undo operation ({record.RedoOperation}/{record.UndoOperation})"; + return null; + } + + var lcnCount = U16(data, 0x0E); + if (HeaderLength + (8 * lcnCount) > data.Length) + { + error = $"log record {lfs.Lsn:X} lists more clusters than it holds"; + return null; + } + + record.Lcns = new long[lcnCount]; + for (var i = 0; i < lcnCount; i++) + { + record.Lcns[i] = (long)U64(data, HeaderLength + (8 * i)); + } + + // Data offsets are 8-byte aligned and start after the header and clusters; data may end early (see RedoData). + var minimum = HeaderLength + (8 * lcnCount); + if ((record.RedoLength > 0 && (record.RedoOffset < minimum || (record.RedoOffset & 7) != 0)) || + (record.UndoLength > 0 && (record.UndoOffset < minimum || (record.UndoOffset & 7) != 0))) + { + error = $"log record {lfs.Lsn:X} has misplaced redo or undo data"; + return null; + } + + return record; + } + +} + +/// +/// The NTFS restart record (written at every checkpoint): where analysis starts and where the dumps of the +/// four restart tables are. +/// +internal sealed class NtfsRestartRecord +{ + public uint MajorVersion { get; private set; } + + public uint MinorVersion { get; private set; } + + public ulong CheckpointLsn { get; private set; } + + public ulong OpenAttributeTableLsn { get; private set; } + + public ulong AttributeNamesLsn { get; private set; } + + public ulong DirtyPageTableLsn { get; private set; } + + public ulong TransactionTableLsn { get; private set; } + + public int OpenAttributeTableLength { get; private set; } + + public int AttributeNamesLength { get; private set; } + + public int DirtyPageTableLength { get; private set; } + + public int TransactionTableLength { get; private set; } + + public static NtfsRestartRecord Parse(LfsRecord record) + { + var d = record.ClientData; + if (record.RecordType != LfsRecord.TypeClientRestart || d.Length < 0x40) + { + throw new JournalException($"The NTFS restart record at {record.Lsn:X} is not a restart record"); + } + + var result = new NtfsRestartRecord + { + MajorVersion = U32(d, 0x00), + MinorVersion = U32(d, 0x04), + CheckpointLsn = U64(d, 0x08), + OpenAttributeTableLsn = U64(d, 0x10), + AttributeNamesLsn = U64(d, 0x18), + DirtyPageTableLsn = U64(d, 0x20), + TransactionTableLsn = U64(d, 0x28), + OpenAttributeTableLength = (int)U32(d, 0x30), + AttributeNamesLength = (int)U32(d, 0x34), + DirtyPageTableLength = (int)U32(d, 0x38), + TransactionTableLength = (int)U32(d, 0x3C), + }; + + if (result.MajorVersion > 1) + { + throw new JournalException($"NTFS restart version {result.MajorVersion}.{result.MinorVersion} is not supported"); + } + + if (result.OpenAttributeTableLength < 0 || result.AttributeNamesLength < 0 || + result.DirtyPageTableLength < 0 || result.TransactionTableLength < 0) + { + throw new JournalException($"The NTFS restart record at {record.Lsn:X} has an invalid table length"); + } + + return result; + } +} + +/// +/// An entry of the open attribute table: an attribute that log records refer to by the entry's offset. +/// +internal sealed class OpenAttributeEntry +{ + public const int SizeVersion0 = 0x2C; + public const int SizeVersion1 = 0x28; + + public uint AttributeType { get; set; } + + public ulong FileReference { get; set; } + + public uint BytesPerIndexBuffer { get; set; } + + public ulong OpenRecordLsn { get; set; } + + public string Name { get; set; } = ""; + + public bool IsIndexAllocation => AttributeType == 0xA0; + + /// Reads an entry in the layout of NTFS restart version 0 or 1. + public static OpenAttributeEntry Parse(byte[] b, int offset, uint version) + { + if (version == 0) + { + return new OpenAttributeEntry + { + FileReference = U64(b, offset + 0x08), + OpenRecordLsn = U64(b, offset + 0x10), + AttributeType = U32(b, offset + 0x1C), + BytesPerIndexBuffer = U32(b, offset + 0x28), + }; + } + + return new OpenAttributeEntry + { + BytesPerIndexBuffer = U32(b, offset + 0x04), + AttributeType = U32(b, offset + 0x08), + FileReference = U64(b, offset + 0x10), + OpenRecordLsn = U64(b, offset + 0x18), + }; + } +} + +/// +/// An entry of the dirty page table: a page that may hold changes that are only in the log. +/// +internal sealed class DirtyPageEntry +{ + public uint TargetAttribute { get; set; } + + public uint TransferLength { get; set; } + + public long Vcn { get; set; } + + /// The LSN of the oldest change to this page that may be missing on disk. + public ulong OldestLsn { get; set; } + + /// The page's clusters, one per VCN from ; 0 once the cluster was freed. + public long[] Lcns { get; set; } + + public bool Covers(uint targetAttribute, long vcn) => + TargetAttribute == targetAttribute && vcn >= Vcn && vcn < Vcn + Lcns.Length; + + public static DirtyPageEntry Parse(byte[] b, int offset, int entrySize, uint version) + { + var lcnsOffset = version == 0 ? 0x24 : 0x20; + if (entrySize < lcnsOffset) + { + throw new JournalException("A dirty page table entry is too short"); + } + + var lcnCount = U32(b, offset + 0x0C); + if (lcnCount > (entrySize - lcnsOffset) / 8) + { + throw new JournalException("A dirty page table entry lists more clusters than it holds"); + } + + var vcn = (long)U64(b, offset + (version == 0 ? 0x14 : 0x10)); + if (vcn < 0 || vcn > long.MaxValue - lcnCount) + { + throw new JournalException("A dirty page table entry has an invalid VCN range"); + } + + var entry = new DirtyPageEntry + { + TargetAttribute = U32(b, offset + 0x04), + TransferLength = U32(b, offset + 0x08), + Vcn = vcn, + OldestLsn = U64(b, offset + (version == 0 ? 0x1C : 0x18)), + Lcns = new long[(int)lcnCount], + }; + for (var i = 0; i < lcnCount; i++) + { + entry.Lcns[i] = (long)U64(b, offset + lcnsOffset + (8 * i)); + } + + return entry; + } +} + +internal enum TransactionState : byte +{ + Uninitialized = 0, + Active = 1, + Prepared = 2, + Committed = 3, +} + +/// +/// An entry of the transaction table; its offset in the table is the transaction id in log records. +/// +internal sealed class TransactionEntry +{ + public const int Size = 0x20; + + public TransactionState State { get; set; } + + public ulong FirstLsn { get; set; } + + public ulong PreviousLsn { get; set; } + + /// The next record to undo when rolling the transaction back; 0 if there is nothing to undo. + public ulong UndoNextLsn { get; set; } + + public static TransactionEntry Parse(byte[] b, int offset) => new() + { + State = (TransactionState)b[offset + 0x04], + FirstLsn = U64(b, offset + 0x08), + PreviousLsn = U64(b, offset + 0x10), + UndoNextLsn = U64(b, offset + 0x18), + }; +} + +/// +/// Reads the restart tables that NTFS dumps into the log at every checkpoint. +/// +internal static class RestartTable +{ + private const int HeaderLength = 0x18; + private const uint EntryAllocated = 0xFFFFFFFF; + + /// + /// The allocated entries of a dumped restart table, keyed by their byte offset from the start of the table. + /// + public static IEnumerable<(int Offset, int EntrySize)> Entries(byte[] table) + { + if (table.Length < HeaderLength) + { + throw new JournalException("A restart table dump is too short"); + } + + int entrySize = U16(table, 0x00); + int entryCount = U16(table, 0x02); + if (entrySize < 8 || entryCount > (table.Length - HeaderLength) / entrySize) + { + throw new JournalException("A restart table dump has an implausible header"); + } + + for (var i = 0; i < entryCount; i++) + { + var offset = HeaderLength + (i * entrySize); + if (U32(table, offset) == EntryAllocated) + { + yield return (offset, entrySize); + } + } + } + + /// + /// The attribute names dump: (open attribute table offset, name) pairs. + /// + public static IEnumerable<(int Offset, string Name)> AttributeNames(byte[] dump) + { + var position = 0; + while (position <= dump.Length - 4) + { + int tableOffset = U16(dump, position); + int nameBytes = U16(dump, position + 2); + if (tableOffset == 0 && nameBytes == 0) + { + yield break; + } + + if (nameBytes > dump.Length - position - 4) + { + throw new JournalException("The attribute names dump is damaged"); + } + + yield return (tableOffset, Encoding.Unicode.GetString(dump, position + 4, nameBytes)); + position += 4 + nameBytes + 2; + } + } +} diff --git a/Library/DiscUtils.Ntfs/Journal/NtfsLogReplayer.cs b/Library/DiscUtils.Ntfs/Journal/NtfsLogReplayer.cs new file mode 100644 index 000000000..043d30d04 --- /dev/null +++ b/Library/DiscUtils.Ntfs/Journal/NtfsLogReplayer.cs @@ -0,0 +1,485 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.IO; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs.Journal; + +/// +/// The redo and undo passes of NTFS recovery, applied to an in-memory set of pages. +/// +/// +/// Redo repeats history: every change after a page's oldest dirty LSN is applied again when the page (file +/// record or index buffer) carries an older LSN than the record; pages without an LSN (bitmaps, plain data) are +/// replayed unconditionally, which is safe because those operations set bytes and bits to fixed values. Undo then +/// follows each active transaction's undo-next chain backwards. Undo of different transactions is supported +/// only for disjoint clusters, except that bitmap bit runs only need to be disjoint: setting or clearing different +/// bits of one cluster gives the same result in any order. We do not write compensation records or persist changes +/// that require undo. +/// +internal sealed class NtfsLogReplayer +{ + private readonly NtfsLogAnalysis _analysis; + private readonly JournalVolumeLayout _layout; + private readonly NtfsLogPages _pages; + + public NtfsLogReplayer(Stream volume, JournalVolumeLayout layout, NtfsLogAnalysis analysis) + { + _analysis = analysis; + _layout = layout; + _pages = new NtfsLogPages(volume, layout); + } + + public NtfsLogPages Pages => _pages; + + /// Log records whose redo action changed a page. + public int RedoneRecords { get; private set; } + + /// Log records whose undo action changed a page. + public int UndoneRecords { get; private set; } + + public int UndoneTransactions => _pages.UndoTransactionCount; + + /// The MFT record numbers that changed (as far as the log tells, for reporting). + public SortedSet ChangedFileRecords { get; } = []; + + public void Redo() + { + if (_analysis.RedoLsn == 0) + { + return; + } + + var foundStart = false; + foreach (var lfs in _analysis.Log.ReadForward(_analysis.RedoLsn)) + { + foundStart = true; + if (!_analysis.Log.IsForNtfsClient(lfs) || lfs.RecordType != LfsRecord.TypeClientRecord) + { + continue; + } + + var record = NtfsLogRecord.TryParse(lfs, out var error) ?? throw new JournalException(error); + var target = NtfsLogOperations.TargetOf(record.RedoOperation); + if (target == NtfsLogTarget.None) + { + continue; + } + + if (record.Lcns.Length == 0) + { + if (record.UndoOperation == NtfsLogOperation.CompensationLogRecord) + { + throw new JournalException($"Compensation record {record.Lsn:X} changes metadata without a cluster mapping"); + } + + continue; + } + + var page = _analysis.FindDirtyPage(record.TargetAttribute, record.TargetVcn); + if (page == null || record.Lsn < _analysis.OldestLsnFor(record.TargetAttribute, record.TargetVcn)) + { + continue; + } + + // The dirty page table has the current clusters (DeleteDirtyClusters zeroes freed ones). + var first = (int)(record.TargetVcn - page.Vcn); + var lcns = new long[page.Lcns.Length - first]; + Array.Copy(page.Lcns, first, lcns, 0, lcns.Length); + if (Apply(record, record.RedoOperation, record.RedoData, lcns, redo: true)) + { + RedoneRecords++; + } + } + + if (!foundStart && _analysis.DirtyPages.Count != 0) + { + throw new JournalException($"The redo start record {_analysis.RedoLsn:X} is not in the log"); + } + } + + public void Undo() + { + foreach (var loser in _analysis.Losers) + { + var lsn = loser.Value.UndoNextLsn; + while (lsn != 0) + { + var lfs = _analysis.Log.ReadRecord(lsn) + ?? throw new JournalException($"Record {lsn:X} to undo is not in the log"); + _analysis.Log.RequireNtfsRecord(lfs, "undo chain record", LfsRecord.TypeClientRecord); + + if (lfs.TransactionId != loser.Key) + { + throw new JournalException($"Record {lsn:X} in the undo chain of transaction {loser.Key:X} belongs to {lfs.TransactionId:X}"); + } + + if (lsn < loser.Value.FirstLsn || lsn > loser.Value.PreviousLsn || lsn > _analysis.LastLsn) + { + throw new JournalException($"Record {lsn:X} is outside the undo history of transaction {loser.Key:X}"); + } + + var next = lfs.ClientUndoNextLsn; + if (next != 0 && next >= lsn) + { + throw new JournalException($"The undo chain of transaction {loser.Key:X} loops or points forward at {lsn:X}"); + } + + var record = NtfsLogRecord.TryParse(lfs, out var error) ?? throw new JournalException(error); + var target = NtfsLogOperations.TargetOf(record.UndoOperation); + if (target != NtfsLogTarget.None) + { + if (record.Lcns.Length == 0) + { + throw new JournalException($"Record {lsn:X} needs metadata undo without a cluster mapping"); + } + + var lcns = (long[])record.Lcns.Clone(); + + // A cluster the dirty page table says was freed holds nothing of this page any more. + var page = _analysis.FindDirtyPage(record.TargetAttribute, record.TargetVcn); + if (page != null) + { + for (var i = 0; i < lcns.Length && record.TargetVcn - page.Vcn + i < page.Lcns.Length; i++) + { + var current = page.Lcns[record.TargetVcn - page.Vcn + i]; + if (current == 0) + { + lcns[i] = 0; + } + else if (current != lcns[i]) + { + throw new JournalException($"Record {lsn:X} has a remapped cluster in its undo target"); + } + } + } + + if (Apply(record, record.UndoOperation, record.UndoData, lcns, redo: false)) + { + UndoneRecords++; + } + } + + // A compensation record points past the changes that were already undone. + lsn = next; + } + } + } + + /// + /// Applies one action; false if it was not needed (the page is newer, or its clusters were freed). + /// + private bool Apply(NtfsLogRecord record, NtfsLogOperation op, ArraySegment data, long[] lcns, bool redo) + { + var target = NtfsLogOperations.TargetOf(op); + var start = record.ClusterBlockOffset * 512; + var undoTransaction = redo ? 0 : record.Lfs.TransactionId; + switch (target) + { + case NtfsLogTarget.FileRecord: + { + if (record.TargetVcn < 0 || record.TargetVcn > (long.MaxValue - start) / _layout.BytesPerCluster) + { + throw new JournalException($"Log record {record.Lsn:X} has an invalid file record VCN"); + } + + var length = _layout.MftRecordSize; + if (!Allocated(lcns, start, length)) + { + return false; + } + + var page = _pages.GetLogical(PageKind.FileRecord, lcns, start, length, undoTransaction); + if (!PageNeedsAction(page, record, redo)) + { + return false; + } + + var torn = !page.Valid && page.Signature == SignatureFile; + if (!page.Valid) + { + // A record that was never written can be rebuilt (InitializeFileRecordSegment) or freed again + // (DeallocateFileRecordSegment, the compensation of a creation that never reached the disk; + // Windows then only changes the header fields). Anything else needs a real record. + if (op == NtfsLogOperation.InitializeFileRecordSegment) + { + Array.Clear(page.Data, 0, page.Data.Length); + } + else if (op != NtfsLogOperation.DeallocateFileRecordSegment || page.Signature == SignatureFile) + { + throw new JournalException($"Log record {record.Lsn:X} changes file record at cluster {lcns[start / _pages.ClusterSize]}, which is damaged"); + } + } + + var lsn = redo ? record.Lsn : page.Lsn; + NtfsLogActions.ApplyToFileRecord(op, record, page.Data, data); + Finish(page, lsn, stampInvalid: redo); + if (torn) + { + var used = U32(page.Data, 0x18); + var firstAttribute = U16(page.Data, 0x14); + if (!page.Valid || record.RecordOffset != 0 || firstAttribute < 0x20 || used < firstAttribute + 4 || data.Count < used) + { + throw new JournalException($"Log record {record.Lsn:X} does not supply a complete replacement for the torn file record"); + } + } + + ChangedFileRecords.Add(((record.TargetVcn * _layout.BytesPerCluster) + start) / _layout.MftRecordSize); + return true; + } + + case NtfsLogTarget.IndexBuffer: + { + var length = IndexBufferSize(record); + start += record.RecordOffset; + if (!Allocated(lcns, start, length)) + { + return false; + } + + var page = _pages.GetLogical(PageKind.IndexBuffer, lcns, start, length, undoTransaction); + if (!PageNeedsAction(page, record, redo)) + { + return false; + } + + if (!page.Valid) + { + throw new JournalException($"Log record {record.Lsn:X} changes an index buffer at cluster {lcns[start / _pages.ClusterSize]}, which is damaged"); + } + + var lsn = redo ? record.Lsn : page.Lsn; + NtfsLogActions.ApplyToIndexBuffer(op, record, page.Data, data); + Finish(page, lsn, stampInvalid: redo); + return true; + } + + case NtfsLogTarget.NonResident: + return ApplyNonResident(record, op, data, lcns, start, redo, undoTransaction); + + default: + throw NtfsLogActions.Unsupported(op, record); + } + } + + private bool ApplyNonResident(NtfsLogRecord record, NtfsLogOperation op, ArraySegment data, long[] lcns, int start, + bool redo, uint undoTransaction) + { + if (!_analysis.OpenAttributes.TryGetValue(record.TargetAttribute, out var attribute)) + { + throw new JournalException($"Log record {record.Lsn:X} changes attribute {record.TargetAttribute:X}, which is not open"); + } + + if (op == NtfsLogOperation.UpdateNonresidentValue) + { + if (attribute.IsIndexAllocation) + { + // Writes into an index buffer (often a whole new one); the bytes are in logical form. + var length = IndexBufferSize(record); + if (!Allocated(lcns, start, length)) + { + return false; + } + + var page = _pages.GetLogical(PageKind.IndexBuffer, lcns, start, length, undoTransaction); + if (record.RecordOffset + data.Count > length) + { + throw new JournalException($"Log record {record.Lsn:X} writes past its index buffer"); + } + + if ((page.Signature == SignatureIndx || page.Signature == SignatureBaad || page.Signature == SignatureHole) && + !PageNeedsAction(page, record, redo)) + { + return false; + } + + var torn = !page.Valid && page.Signature == SignatureIndx; + if (torn && (record.RecordOffset != 0 || data.Count != length)) + { + throw new JournalException($"Log record {record.Lsn:X} only partially replaces a torn index buffer"); + } + + var lsn = redo ? record.Lsn : page.Lsn; + Array.Copy(data.Array, data.Offset, page.Data, record.RecordOffset, data.Count); + Finish(page, lsn, stampInvalid: false); + if (torn && !page.Valid) + { + throw new JournalException($"Log record {record.Lsn:X} supplies an invalid replacement for the torn index buffer"); + } + + return true; + } + + var offset = start + record.RecordOffset; + if (!Allocated(lcns, offset, Math.Max(1, data.Count))) + { + return false; + } + + if (data.Count == 0) + { + return false; + } + + var current = _pages.ReadRaw(lcns, offset, data.Count, undoTransaction); + if (current.AsSpan().SequenceEqual(data.AsSpan())) + { + return false; + } + + _pages.WriteRaw(lcns, offset, data.Array, data.Offset, data.Count); + return true; + } + + // Set or clear a run of bits; the record's data is the first bit and the count. + if (data.Count < 8) + { + throw new JournalException($"Log record {record.Lsn:X} has a bit range that is too short"); + } + + var firstBit = (long)U32(data.Array, data.Offset); + var bits = (long)U32(data.Array, data.Offset + 4); + var byteStart = start + record.RecordOffset + (int)(firstBit >> 3); + var byteCount = (int)(((firstBit & 7) + bits + 7) >> 3); + if (bits == 0) + { + return false; + } + + if (bits > int.MaxValue - (firstBit & 7)) + { + throw new JournalException($"Log record {record.Lsn:X} has an oversized bit range"); + } + + if (!Allocated(lcns, byteStart, byteCount)) + { + return false; + } + + var bytes = _pages.ReadBits(lcns, byteStart, (int)(firstBit & 7), (int)bits, undoTransaction); + var before = (byte[])bytes.Clone(); + NtfsLogActions.ChangeBits(bytes, (int)(firstBit & 7), (int)bits, op == NtfsLogOperation.SetBitsInNonresidentBitMap); + if (before.AsSpan().SequenceEqual(bytes)) + { + return false; + } + + _pages.WriteRaw(lcns, byteStart, bytes, 0, byteCount); + return true; + } + + /// + /// Redo checks the page LSN. Undo checks that the page history is plausible, then follows the chain without + /// using a page stamp as an undo cursor. More than one inverse operation can target the same page. + /// + private bool PageNeedsAction(LogicalPage page, NtfsLogRecord record, bool redo) + { + if (page.Signature == SignatureBaad) + { + throw new JournalException($"Log record {record.Lsn:X} changes a page chkdsk marked bad"); + } + + if (page.Signature == SignatureHole) + { + return false; + } + + // A torn record (signature, but the update sequence array does not match) has no trustworthy LSN. + if (!page.Valid && (page.Signature == SignatureFile || page.Signature == SignatureIndx)) + { + return true; + } + + if (redo) + { + return page.Lsn < record.Lsn; + } + + if (page.Valid && (page.Lsn < record.Lsn || page.Lsn > _analysis.LastLsn)) + { + throw new JournalException($"Page LSN {page.Lsn:X} cannot establish the undo history for record {record.Lsn:X}"); + } + + return true; + } + + private static void Finish(LogicalPage page, ulong lsn, bool stampInvalid) + { + // An undo can turn a page back into "never written" (no signature); it is then written as it is. + var expected = page.Kind == PageKind.FileRecord ? SignatureFile : SignatureIndx; + page.Valid = page.Signature == expected && HasFixupLayout(page.Data, 0, page.Data.Length); + page.Dirty = true; + + // In-memory undo retains the page's historical LSN. It does not create a new log record. + if (stampInvalid || page.Valid) + { + page.Lsn = lsn; + } + } + + private int IndexBufferSize(NtfsLogRecord record) + { + if (_analysis.OpenAttributes.TryGetValue(record.TargetAttribute, out var attribute) && attribute.BytesPerIndexBuffer != 0) + { + if (!JournalVolumeLayout.IsRecordSize(attribute.BytesPerIndexBuffer)) + { + throw new JournalException($"Log record {record.Lsn:X} has an invalid index buffer size"); + } + + return (int)attribute.BytesPerIndexBuffer; + } + + return _layout.IndexBufferSize; + } + + /// + /// false if the first cluster of the range was freed (nothing to do); a range that is only partly freed + /// cannot be replayed faithfully. + /// + private bool Allocated(long[] lcns, int start, int length) + { + var first = start / _pages.ClusterSize; + var last = (start + (long)length - 1) / _pages.ClusterSize; + if (start < 0 || length <= 0 || last >= lcns.Length) + { + throw new JournalException("A log record addresses bytes beyond the clusters it names"); + } + + if (lcns[first] == 0) + { + return false; + } + + for (var i = first + 1; i <= last; i++) + { + if (lcns[i] == 0) + { + throw new JournalException("A log record changes a page whose clusters were partly freed"); + } + } + + return true; + } +} diff --git a/Library/DiscUtils.Ntfs/NtfsFileSystem.cs b/Library/DiscUtils.Ntfs/NtfsFileSystem.cs index 7c3bd5971..db2e98330 100644 --- a/Library/DiscUtils.Ntfs/NtfsFileSystem.cs +++ b/Library/DiscUtils.Ntfs/NtfsFileSystem.cs @@ -63,9 +63,37 @@ public class NtfsFileSystem : DiscFileSystem, IClusterBasedFileSystem, /// Initializes a new instance of the NtfsFileSystem class. /// /// The stream containing the NTFS file system. + /// + /// A writable stream requires a clean journal. On a read-only stream, journal replay is applied in memory + /// when supported. Use the overload with to authorize writes at open. + /// When opening through , set the entry + /// NtfsJournalMode to Replay, Refuse or Ignore to override the default for that open. + /// public NtfsFileSystem(Stream stream) + : this(stream, stream?.CanWrite == true ? NtfsJournalMode.Refuse : NtfsJournalMode.Replay) + { + } + + /// + /// Initializes a new instance of the NtfsFileSystem class. + /// + /// The stream containing the NTFS file system. + /// What to do with the journal of a volume Windows did not shut down cleanly. + /// The journal needs a replay that does not allow, + /// or that DiscUtils cannot do on a writable stream. + /// is not a defined mode. + /// + public NtfsFileSystem(Stream stream, NtfsJournalMode journalMode) : base(new NtfsOptions()) { + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + + stream = HandleJournal(stream, journalMode, out var journal); + JournalInfo = journal; + _context = new NtfsContext { RawStream = stream, @@ -153,21 +181,51 @@ public NtfsFileSystem(Stream stream) } } -#if false - byte[] buffer = new byte[1024]; - for (int i = 0; i < buffer.Length; ++i) - { - buffer[i] = 0xFF; - } + } + + /// + /// Gets what was found in the journal when the volume was opened, and whether it was replayed; + /// null with . + /// + public NtfsJournalInfo JournalInfo { get; } - using (var s = OpenFile("$LogFile", FileMode.Open, FileAccess.ReadWrite)) + private static Stream HandleJournal(Stream stream, NtfsJournalMode journalMode, out NtfsJournalInfo info) + { + info = null; + switch (journalMode) { - while (s.Position != s.Length) - { - s.Write(buffer, 0, (int)Math.Min(buffer.Length, s.Length - s.Position)); - } + case NtfsJournalMode.Ignore: + return stream; + + case NtfsJournalMode.Refuse: + info = NtfsJournal.Analyze(stream); + if (info.Status == NtfsJournalStatus.Unsupported) + { + throw new IOException($"The NTFS journal cannot be replayed: {info.Detail}"); + } + + if (info.Status == NtfsJournalStatus.NeedsReplay || (stream.CanWrite && info.Status == NtfsJournalStatus.NothingToReplay)) + { + throw new IOException($"The NTFS journal must be replayed or reset before this volume can be opened ({info.Detail})"); + } + + return stream; + + case NtfsJournalMode.Replay: + if (stream.CanWrite) + { + // Like Windows at mount: replay onto the volume, then reset the journal. + info = NtfsJournal.Replay(stream); + return stream; + } + + // Read-only: show the replayed metadata from memory, never write the volume. + info = NtfsJournal.ReplayInMemory(stream, out var inMemory); + return inMemory ?? stream; + + default: + throw new ArgumentOutOfRangeException(nameof(journalMode), journalMode, "Unknown NTFS journal mode"); } -#endif } private bool CreateShortNames => _context.Options.ShortNameCreation == ShortFileNameOption.Enabled @@ -2144,7 +2202,8 @@ protected override void Dispose(bool disposing) _context.Mft = null; } - if (_context.Options.Compressor is IDisposable disposableCompressor) + // _context is null when the constructor threw (e.g. the journal needed a replay that was refused). + if (_context?.Options.Compressor is IDisposable disposableCompressor) { if (disposing) { diff --git a/Library/DiscUtils.Ntfs/NtfsFormatter.cs b/Library/DiscUtils.Ntfs/NtfsFormatter.cs index 35476ea27..2c212aa41 100644 --- a/Library/DiscUtils.Ntfs/NtfsFormatter.cs +++ b/Library/DiscUtils.Ntfs/NtfsFormatter.cs @@ -229,7 +229,8 @@ public NtfsFileSystem Format(Stream stream) } // XP-style security permissions setup - var ntfs = new NtfsFileSystem(stream); + // The volume is still being built; its journal is empty anyway. + var ntfs = new NtfsFileSystem(stream, NtfsJournalMode.Ignore); ntfs.SetSecurity(@"$MFT", new RawSecurityDescriptor($"O:{localAdminString}G:BAD:(A;;FR;;;SY)(A;;FR;;;BA)")); diff --git a/Library/DiscUtils.Ntfs/NtfsJournal.cs b/Library/DiscUtils.Ntfs/NtfsJournal.cs new file mode 100644 index 000000000..83f3719a0 --- /dev/null +++ b/Library/DiscUtils.Ntfs/NtfsJournal.cs @@ -0,0 +1,535 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace DiscUtils.Ntfs; + +/// +/// The state of an NTFS volume's journal ($LogFile). +/// +public enum NtfsJournalStatus +{ + /// + /// Windows shut the volume down cleanly, or the journal is empty or was reset (also by chkdsk). + /// + Clean, + + /// + /// The journal was left open, but replay would make no metadata changes. An explicit replay resets it. + /// + NothingToReplay, + + /// + /// The journal holds changes that are missing on disk, or unfinished transactions that must be rolled back. + /// Until the journal is replayed, the volume's metadata is out of date and later recovery may overwrite + /// intervening writes. + /// + NeedsReplay, + + /// + /// The journal cannot be replayed by DiscUtils (unsupported version or operation, or damage); see + /// . + /// + Unsupported, +} + +/// +/// What to do with the journal when an is opened. +/// +public enum NtfsJournalMode +{ + /// + /// Replay supported journal changes. On a writable stream, replay writes metadata and resets the journal + /// only when is true. On a read-only stream, replay uses + /// an in-memory overlay. A journal DiscUtils cannot replay makes opening a writable stream fail with an + /// . A read-only stream is opened as it is when recovery is refused, including + /// when there is local hibernation evidence or the hibernation check is indeterminate. + /// + Replay, + + /// + /// Require a clean journal for a writable open. A read-only open also accepts + /// . Other states cause an . + /// + Refuse, + + /// + /// Do not look at the journal (the behavior of DiscUtils before journal support). A volume that needs a + /// replay then shows out-of-date metadata, and changes to it can be overwritten by Windows' own replay. + /// + Ignore, +} + +/// +/// The result of looking at (and possibly replaying) an NTFS journal. +/// +public sealed class NtfsJournalInfo +{ + internal NtfsJournalInfo(NtfsJournalStatus status, string detail) + { + Status = status; + Detail = detail; + } + + /// Gets the state found during analysis, before any replay or journal reset. + public NtfsJournalStatus Status { get; internal set; } + + /// Gets a human-readable explanation (for , why). + public string Detail { get; internal set; } + + /// Gets the log file service version, e.g. "2.0" (empty when there is no log). + public string LogVersion { get; internal set; } = ""; + + /// Gets the LSN of the last record in the log (0 when there is no log to read). + public ulong LastLsn { get; internal set; } + + /// Gets the number of log records after the last checkpoint. + public int RecordsAfterCheckpoint { get; internal set; } + + /// Gets the number of log records the redo pass applies (0 when replaying changes nothing). + public int RecordsToRedo { get; internal set; } + + /// Gets the number of unfinished transactions with undo actions targeting metadata. + public int TransactionsToUndo { get; internal set; } + + /// Gets the number of log records the undo pass applies. + public int RecordsToUndo { get; internal set; } + + /// Gets the number of file records, index buffers and other pages the replay changes. + public int PagesChanged { get; internal set; } + + /// Gets the MFT record numbers the replay changes. + public IReadOnlyList FileRecordsChanged { get; internal set; } = []; + + /// + /// Gets a value indicating whether replay wrote to the volume stream, reset the journal and completed its flush calls. + /// This is also true for a reset with no metadata changes (). + /// + public bool Replayed { get; internal set; } + + /// Gets a value indicating whether the read-only mount uses an in-memory replay overlay. + public bool ReplayedInMemory { get; internal set; } + + /// + /// Gets whether this journal is eligible for persistent replay. Metadata undo requires compensation records + /// for recoverable on-disk replay and is available only in memory. Resetting a log with multiple client slots + /// is unsupported. Stream capabilities and exclusive ownership must be established by the caller. + /// + public bool CanReplayPersistently => Status != NtfsJournalStatus.Unsupported && TransactionsToUndo == 0 && !HasMultipleClientSlots; + + internal bool HasMultipleClientSlots { get; set; } + + /// + public override string ToString() => + $"{Status}: {Detail} (LFS {LogVersion}, {RecordsAfterCheckpoint} records after the checkpoint, redo {RecordsToRedo}, " + + $"undo {RecordsToUndo} in {TransactionsToUndo} transactions, {PagesChanged} pages)"; +} + +/// +/// Reads and replays supported journal ($LogFile) records on an NTFS volume after an unclean shutdown. +/// +/// +/// When a journal needs replay or reset, recovery checks the local root hibernation file before exposing an +/// overlay or writing to the volume. Evidence of a saved Windows resume image or an indeterminate check prevents +/// recovery. An empty or clean journal needs no recovery and is returned without this check. +/// Use an offline volume with exclusive access. A suspended Windows installation can still own a data volume +/// whose hibernation file is on a different volume. The local check cannot establish that the volume was released. +/// +public static class NtfsJournal +{ + /// + /// Looks at the journal without writing anything: whether the volume needs a replay, and what it would change. + /// + /// The NTFS volume (the partition, not the whole disk). + public static NtfsJournalInfo Analyze(Stream volume) => Run(volume, write: false, out _, out _); + + /// + /// Replays the journal onto the volume and resets it. Does nothing if the journal is clean. + /// + /// The NTFS volume (the partition, not the whole disk); must be writable. + /// + /// Replay prepares the changes before writing. If the MFT mirror changes, it is written and flushed before + /// the primary records. Replay flushes metadata before resetting the restart pages. It writes and flushes + /// the alternate restart page before replacing and flushing the source page. + /// Replay requests with true through a non-owning sparse + /// stream adapter. Built-in wrappers forward the request to backing streams that support disk flushing; + /// other streams retain their parameterless behavior. The caller must establish + /// whether every storage layer honors the barrier. + /// Replay is not atomic. An I/O error during the writes or flushes can leave a partially updated volume. + /// Journals that require metadata undo or fail the local hibernation check cannot be written by this API. + /// + /// The stream is read-only, the journal is unsupported (before writing), or an I/O error occurred. + public static NtfsJournalInfo Replay(Stream volume) + { + var info = Run(volume, write: true, out _, out _); + if (info.Status == NtfsJournalStatus.Unsupported) + { + throw new IOException($"The NTFS journal cannot be replayed: {info.Detail}"); + } + + return info; + } + + /// + /// Replays the journal into memory only: the result is a stream over that shows the + /// replayed metadata, while itself is never written. + /// + internal static NtfsJournalInfo ReplayInMemory(Stream volume, out SparseStream replayed) + { + var info = Run(volume, write: false, out var clusters, out var clusterSize); + replayed = null; + if (info.Status == NtfsJournalStatus.NeedsReplay && clusters != null) + { + replayed = CreateReplayOverlay(volume, clusters, clusterSize); + info.ReplayedInMemory = true; + } + + return info; + } + + private static NtfsJournalInfo Run(Stream volume, bool write, out Dictionary changedClusters, out int clusterSize) + { + if (volume == null) + { + throw new ArgumentNullException(nameof(volume)); + } + + if (!volume.CanRead || !volume.CanSeek) + { + throw new ArgumentException("Reading the NTFS journal needs a readable, seekable stream", nameof(volume)); + } + + if (write && !volume.CanWrite) + { + throw new IOException("Replaying the NTFS journal needs a writable stream"); + } + + changedClusters = null; + clusterSize = 0; + JournalVolumeLayout layout; + LfsLog log; + LfsLog.Kind kind; + LogFileStorage storage; + try + { + layout = JournalVolumeLayout.Read(volume); + clusterSize = layout.BytesPerCluster; + storage = LogFileStorage.Open(volume, layout); + log = LfsLog.Open(storage, out kind); + } + catch (JournalException e) + { + return new NtfsJournalInfo(NtfsJournalStatus.Unsupported, e.Message); + } + + if (kind == LfsLog.Kind.Empty) + { + return new NtfsJournalInfo(NtfsJournalStatus.Clean, "the journal is empty (freshly formatted or reset)"); + } + + if (kind == LfsLog.Kind.CheckedByChkdsk) + { + return new NtfsJournalInfo(NtfsJournalStatus.Clean, "chkdsk repaired the volume and obsoleted the journal"); + } + + var info = new NtfsJournalInfo(NtfsJournalStatus.Clean, "") + { + LogVersion = $"{log.Restart.MajorVersion}.{log.Restart.MinorVersion}", + }; + + if (!log.Restart.RequiresRecovery) + { + info.Detail = "the journal is marked cleanly closed or has no client in use"; + return info; + } + + Dictionary clusters; + byte[] alternateRestart = null; + byte[] sourceRestart = null; + var sourceOffset = log.Restart.Offset; + var alternateOffset = log.Restart.SystemPageSize - sourceOffset; + long mirrorClusterCount; + try + { + var analysis = NtfsLogAnalysis.Run(log, layout); + info.RecordsAfterCheckpoint = analysis.RecordsAfterCheckpoint; + info.LastLsn = analysis.LastLsn; + var replayer = new NtfsLogReplayer(volume, layout, analysis); + replayer.Redo(); + replayer.Undo(); + info.RecordsToRedo = replayer.RedoneRecords; + info.RecordsToUndo = replayer.UndoneRecords; + info.TransactionsToUndo = replayer.UndoneTransactions; + info.FileRecordsChanged = [.. replayer.ChangedFileRecords]; + info.HasMultipleClientSlots = log.Restart.LogClients != 1; + if (write && !info.CanReplayPersistently) + { + // The reset image closes the log and puts slot zero on the free list. + throw new JournalException(info.HasMultipleClientSlots ? "Persistent replay can only reset a log with a single LFS client slot" : + "Persistent replay requires metadata undo, which DiscUtils supports only in memory"); + } + + clusters = replayer.Pages.Commit(); + mirrorClusterCount = AddMftMirror(volume, layout, clusters); + if (clusters.Keys.Any(storage.ContainsCluster)) + { + throw new JournalException("A metadata replay write overlaps $LogFile"); + } + + if (log.PublishedReset is { } published && + (published.CurrentLsn != Math.Max(analysis.LastLsn, log.Restart.CurrentLsn) || clusters.Count != 0 || !info.CanReplayPersistently)) + { + throw new JournalException("A partial $LogFile reset disagrees with recovery from its remaining dirty page"); + } + + RequireNoLocalHibernationEvidence(volume, layout, allowMftMirror: mirrorClusterCount != 0 && clusters.ContainsKey(layout.MftCluster)); + if (clusters.Count != 0) + { + // Pending metadata could make hiberfil.sys visible or change the clusters it names. + using var overlay = CreateReplayOverlay(volume, clusters, clusterSize); + RequireNoLocalHibernationEvidence(overlay, layout, staged: true); + } + + if (write) + { + if (log.PublishedReset == null) + { + alternateRestart = PrepareLogReset(log, analysis.LastLsn, alternateOffset); + } + + sourceRestart = PrepareLogReset(log, analysis.LastLsn, sourceOffset); + } + + info.PagesChanged = replayer.Pages.ChangedPageCount; + if (clusters.Count == 0) + { + // Replaying changes nothing (bitmap updates, for instance, end where the disk already is). + info.Status = NtfsJournalStatus.NothingToReplay; + info.Detail = log.PublishedReset == null ? "the journal is still open, but replay would not change metadata" : + "one restart page still needs to be reset"; + if (info.TransactionsToUndo == 0) + { + info.RecordsToRedo = 0; + info.RecordsToUndo = 0; + } + + info.FileRecordsChanged = []; + } + else + { + info.Status = NtfsJournalStatus.NeedsReplay; + info.Detail = $"{info.PagesChanged} pages are out of date on disk; {info.TransactionsToUndo} unfinished transactions to roll back"; + } + } + catch (JournalException e) + { + info.Status = NtfsJournalStatus.Unsupported; + info.Detail = e.Message; + return info; + } + + changedClusters = clusters; + if (write) + { + // This adapter owns no resources. Do not dispose it: a non-owning SparseWrapperStream + // performs an extra parameterless flush on disposal, which would change replay ordering + // and could hide the original exception from a failed durability barrier. + var flushStream = SparseStream.FromStream(volume, Ownership.None); + bool IsMirrorCluster(KeyValuePair cluster) => mirrorClusterCount > 0 && + cluster.Key >= layout.MftMirrorCluster && cluster.Key - layout.MftMirrorCluster < mirrorClusterCount; + + if (mirrorClusterCount > 0) + { + // A retry can skip primary records whose LSN already advanced. Their copies must be written first. + WriteClusters(volume, clusters.Where(IsMirrorCluster), clusterSize); + flushStream.Flush(flushToDisk: true); + } + + WriteClusters(volume, clusters.Where(cluster => !IsMirrorCluster(cluster)), clusterSize); + // Flush even without page changes. Metadata or a recognized alternate reset may still be buffered. + flushStream.Flush(flushToDisk: true); + if (alternateRestart != null) + { + storage.Write(alternateOffset, alternateRestart, 0, alternateRestart.Length); + flushStream.Flush(flushToDisk: true); + } + + storage.Write(sourceOffset, sourceRestart, 0, sourceRestart.Length); + flushStream.Flush(flushToDisk: true); + info.Replayed = true; + } + + return info; + } + + private static void RequireNoLocalHibernationEvidence(Stream volume, JournalVolumeLayout layout, + bool allowMftMirror = false, bool staged = false) + { + var (evidence, detail) = NtfsHibernation.Probe(volume, layout, allowMftMirror); + var view = staged ? " in the staged replay view" : ""; + if (evidence == NtfsHibernationEvidence.Active) + { + throw new JournalException($"{detail}{view}. Recovery is refused for hibernation or Fast Startup"); + } + + if (evidence == NtfsHibernationEvidence.Indeterminate) + { + throw new JournalException($"The local Windows hibernation state is indeterminate{view}. {detail}"); + } + } + + private static SnapshotStream CreateReplayOverlay(Stream volume, Dictionary clusters, int clusterSize) + { + var snapshot = new SnapshotStream(new JournalReadOnlyStream(volume), Ownership.Dispose); + snapshot.Snapshot(); + WriteClusters(snapshot, clusters, clusterSize); + snapshot.Freeze(); + return snapshot; + } + + private static void WriteClusters(Stream stream, IEnumerable> clusters, int clusterSize) + { + foreach (var cluster in clusters.OrderBy(c => c.Key)) + { + stream.Position = cluster.Key * clusterSize; + stream.Write(cluster.Value, 0, cluster.Value.Length); + } + } + + /// + /// $MFTMirr holds a copy of the first MFT records; when the replay changes them, the copy follows. + /// + private static long AddMftMirror(Stream volume, JournalVolumeLayout layout, Dictionary clusters) + { + const int reservedRecords = 16; + var minimumMirrorBytes = 4 * layout.MftRecordSize; + var clusterSize = layout.BytesPerCluster; + // A large cluster can mirror additional reserved records. Leave the remaining cluster bytes as stored. + var reservedMirrorBytes = Math.Max(minimumMirrorBytes, Math.Min(clusterSize, reservedRecords * layout.MftRecordSize)); + var mirrorClusters = (reservedMirrorBytes + (long)clusterSize - 1) / clusterSize; + var touched = clusters.Keys.Any(lcn => lcn >= layout.MftCluster && lcn - layout.MftCluster < mirrorClusters); + if (!touched) + { + return 0; + } + + if (!layout.ContainsClusters(layout.MftCluster, mirrorClusters) || + !layout.ContainsClusters(layout.MftMirrorCluster, mirrorClusters) || + Math.Abs(layout.MftCluster - layout.MftMirrorCluster) < mirrorClusters) + { + throw new JournalException("$MFTMirr is outside the volume or overlaps $MFT"); + } + + var mftStart = layout.MftCluster * clusterSize; + var bytes = new byte[reservedMirrorBytes]; + for (var done = 0; done < bytes.Length;) + { + var position = mftStart + done; + var lcn = position / clusterSize; + var inCluster = (int)(position % clusterSize); + var chunk = Math.Min(bytes.Length - done, clusterSize - inCluster); + if (clusters.TryGetValue(lcn, out var cluster)) + { + Array.Copy(cluster, inCluster, bytes, done, chunk); + } + else + { + ReadExactly(volume, position, bytes, done, chunk); + } + + done += chunk; + } + + // The boot sector locates the mirror, but its data attribute must confirm the destination. + var mirrorRecord = new byte[layout.MftRecordSize]; + Array.Copy(bytes, layout.MftRecordSize, mirrorRecord, 0, mirrorRecord.Length); + if (U32(mirrorRecord, 0) != SignatureFile || !RemoveFixups(mirrorRecord, 0, mirrorRecord.Length) || (U16(mirrorRecord, 0x16) & 1) == 0) + { + throw new JournalException("$MFTMirr has no valid base file record"); + } + + var (extents, length) = LogFileStorage.ReadDataAttribute(mirrorRecord, layout, "$MFTMirr"); + if (length < minimumMirrorBytes || extents[0].Lcn != layout.MftMirrorCluster || extents[0].Length < mirrorClusters) + { + throw new JournalException("$MFTMirr data does not match the contiguous mirror location in the boot sector"); + } + + var mirrorBytes = (int)Math.Min(reservedMirrorBytes, length - length % layout.MftRecordSize); + var mirrorStart = layout.MftMirrorCluster * clusterSize; + for (var recordOffset = 0; recordOffset < mirrorBytes; recordOffset += mirrorRecord.Length) + { + Array.Copy(bytes, recordOffset, mirrorRecord, 0, mirrorRecord.Length); + if (U32(mirrorRecord, 0) == SignatureFile && RemoveFixups(mirrorRecord, 0, mirrorRecord.Length)) + { + continue; + } + + // Keep the mirror's copy of a damaged, unmodified primary record. It may be the only valid copy. + ReadExactly(volume, mftStart + recordOffset, mirrorRecord, 0, mirrorRecord.Length); + if (!mirrorRecord.AsSpan().SequenceEqual(bytes.AsSpan(recordOffset, mirrorRecord.Length))) + { + throw new JournalException("A changed file record has no valid image for $MFTMirr"); + } + + ReadExactly(volume, mirrorStart + recordOffset, bytes, recordOffset, mirrorRecord.Length); + } + + for (var done = 0; done < mirrorBytes;) + { + var position = mirrorStart + done; + var lcn = position / clusterSize; + var inCluster = (int)(position % clusterSize); + var chunk = Math.Min(mirrorBytes - done, clusterSize - inCluster); + if (!clusters.TryGetValue(lcn, out var cluster)) + { + cluster = new byte[clusterSize]; + ReadExactly(volume, lcn * clusterSize, cluster, 0, clusterSize); + clusters[lcn] = cluster; + } + + Array.Copy(bytes, done, cluster, inCluster, chunk); + done += chunk; + } + + return mirrorClusters; + } + + /// + /// Leaves an empty journal: both restart pages say log file service 1.1, shut down cleanly, no client in use, + /// with the end of the log as current LSN. Windows starts a new log from there at the next mount. + /// + private static byte[] PrepareLogReset(LfsLog log, ulong lastLsn, long offset) + { + var page = log.Restart.CreateResetImage(lastLsn); + var replaced = new byte[page.Length]; + log.Storage.Read(offset, replaced, 0, replaced.Length); + ApplyFixups(page, 0, page.Length, replaced); + return page; + } +} diff --git a/Library/DiscUtils.Ntfs/README.md b/Library/DiscUtils.Ntfs/README.md index 82e143d9a..eff4b421f 100644 --- a/Library/DiscUtils.Ntfs/README.md +++ b/Library/DiscUtils.Ntfs/README.md @@ -17,11 +17,24 @@ dotnet add package LTRData.DiscUtils.Ntfs | Read files and directories | Yes | | Create / format a filesystem | Yes | | Modify files and directories | Yes, with a writable backing stream | +| NTFS `$LogFile` journal analysis and recovery | Yes, for supported records and safe recovery states | ## Usage notes Supports alternate data streams, security descriptors, reparse points, hard links and allocation metadata through NTFS-specific APIs. Formatting is available through NtfsFileSystem.Format. Support for these APIs does not imply support for every Windows NTFS feature. +### NTFS journal recovery + +DiscUtils can inspect the NTFS `$LogFile` journal after an unclean shutdown and replay supported metadata changes. `NtfsJournal.Analyze(volumeStream)` reports the recovery state without writing; `NtfsFileSystem.JournalInfo` describes what happened when opening a filesystem. + +- **Default writable open:** `new NtfsFileSystem(stream)` refuses a journal requiring recovery or reset, or one it cannot safely interpret. It does not automatically change the volume. +- **Default read-only open:** supported journal changes are replayed in memory, leaving the source unchanged. If recovery is unsupported, the filesystem opens without replay and `JournalInfo.Status` reports `Unsupported`; callers must not assume its metadata is current. +- **Explicit recovery:** `new NtfsFileSystem(stream, NtfsJournalMode.Replay)` or `NtfsJournal.Replay(stream)` performs supported persistent recovery on a writable stream. `NtfsJournalMode.Refuse` rejects volumes requiring recovery; `Ignore` bypasses journal handling and can expose stale metadata. + +**Limitations:** Persistent replay is not atomic and is refused when metadata undo is required; such undo is supported only in memory. The hibernation check covers local evidence, not suspended Windows installations holding the volume from elsewhere. Before persistent replay, ensure the volume is offline and exclusively owned. The `SparseStream.Flush(true)` durability barriers depend on the runtime, all stream wrappers and the underlying storage honoring them. + +See [recovery and undo qualification](../../docs/ntfs-journal-recovery-qualification.md), [hibernation safeguards](../../docs/ntfs-journal-hibernation.md) and [replay durability](../../docs/ntfs-journal-replay-durability.md). + ## Registration For automatic discovery, register the providers implemented by this package: @@ -40,6 +53,7 @@ using DiscUtils.Ntfs; // An existing NTFS volume image, starting at the filesystem boot sector. using var image = File.Open("ntfs.img", FileMode.Open, FileAccess.ReadWrite); +// A writable open requires a clean journal by default; recovery needs explicit opt-in. using var fs = new NtfsFileSystem(image); fs.CreateDirectory("Reports"); using var file = fs.OpenFile(@"Reports\hello.txt", FileMode.Create, FileAccess.Write); diff --git a/Library/DiscUtils.Streams/Block/BlockCacheStream.cs b/Library/DiscUtils.Streams/Block/BlockCacheStream.cs index 13a6e8ffc..7419b1717 100644 --- a/Library/DiscUtils.Streams/Block/BlockCacheStream.cs +++ b/Library/DiscUtils.Streams/Block/BlockCacheStream.cs @@ -571,6 +571,16 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + CheckDisposed(); + + if (CanWrite) + { + _wrappedStream.Flush(flushToDisk); + } + } + /// /// Moves the stream position. /// @@ -814,4 +824,4 @@ private void InvalidateBlocks(long firstBlock, int numBlocks) _cache.ReleaseBlock(i); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/ConcatStream.cs b/Library/DiscUtils.Streams/ConcatStream.cs index 685be13b5..08734fcc4 100644 --- a/Library/DiscUtils.Streams/ConcatStream.cs +++ b/Library/DiscUtils.Streams/ConcatStream.cs @@ -135,6 +135,15 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + CheckDisposed(); + for (var i = 0; i < _streams.Count; ++i) + { + _streams[i].Flush(flushToDisk); + } + } + public override int Read(byte[] buffer, int offset, int count) { CheckDisposed(); @@ -396,4 +405,4 @@ private void CheckDisposed() } #endif } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/MirrorStream.cs b/Library/DiscUtils.Streams/MirrorStream.cs index cce756c8b..0803f9397 100644 --- a/Library/DiscUtils.Streams/MirrorStream.cs +++ b/Library/DiscUtils.Streams/MirrorStream.cs @@ -98,6 +98,14 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + foreach (var stream in _wrapped) + { + stream.Flush(flushToDisk); + } + } + public override int Read(byte[] buffer, int offset, int count) { return _wrapped[0].Read(buffer, offset, count); @@ -240,4 +248,4 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/README.md b/Library/DiscUtils.Streams/README.md index 4d33b036b..6df86b5f0 100644 --- a/Library/DiscUtils.Streams/README.md +++ b/Library/DiscUtils.Streams/README.md @@ -14,6 +14,12 @@ dotnet add package LTRData.DiscUtils.Streams Use this package directly when you need sparse memory streams, stream composition, allocation extents, buffers or stream builders without depending on Core or a filesystem implementation. Ownership controls whether wrappers dispose the streams they wrap. No format registration is needed. +### Storage flush requests + +`SparseStream.Flush(bool flushToDisk)` is a virtual overload for requesting a stronger storage flush. By default it calls `Flush()`. Built-in forwarding wrappers pass this request through nested sparse streams; `SparseStream.FromStream` uses `FileStream.Flush(bool)` when wrapping a `FileStream` and ordinary `Stream.Flush()` for unknown streams. Custom image or remote-storage streams can override this overload to implement an appropriate durability barrier. + +A successful call only establishes what the underlying runtime and storage actually guarantee; it cannot by itself promise power-loss durability or atomic writes. NTFS journal recovery uses `Flush(true)` for its write-ordering barriers. See [NTFS replay durability](../../docs/ntfs-journal-replay-durability.md). + ## Example ```csharp diff --git a/Library/DiscUtils.Streams/SnapshotStream.cs b/Library/DiscUtils.Streams/SnapshotStream.cs index a0ba3c9ee..f7bd2a08b 100644 --- a/Library/DiscUtils.Streams/SnapshotStream.cs +++ b/Library/DiscUtils.Streams/SnapshotStream.cs @@ -253,6 +253,13 @@ public override void Flush() _baseStream.Flush(); } + public override void Flush(bool flushToDisk) + { + CheckFrozen(); + + FlushWrappedStream(_baseStream, flushToDisk); + } + /// /// Flushes the stream. /// @@ -622,4 +629,4 @@ private void CheckFrozen() throw new InvalidOperationException("The stream is frozen"); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/SparseStream.cs b/Library/DiscUtils.Streams/SparseStream.cs index f529cf712..4e4bd8e08 100644 --- a/Library/DiscUtils.Streams/SparseStream.cs +++ b/Library/DiscUtils.Streams/SparseStream.cs @@ -48,6 +48,33 @@ public abstract class SparseStream : CompatibilityStream /// This may be an empty enumeration if all bytes are zero. public abstract IEnumerable Extents { get; } + /// + /// Flushes pending changes, requesting synchronization with backing storage when supported. + /// + /// Whether to request a disk flush from supporting backing streams. + /// + /// The default implementation calls . A stream without a stronger flush + /// contract cannot guarantee that its data has reached durable storage. + /// + public virtual void Flush(bool flushToDisk) => Flush(); + + /// Forwards a flush without bypassing intervening sparse-stream wrappers. + protected static void FlushWrappedStream(Stream wrapped, bool flushToDisk) + { + if (wrapped is SparseStream sparse) + { + sparse.Flush(flushToDisk); + } + else if (wrapped is FileStream file) + { + file.Flush(flushToDisk); + } + else + { + wrapped.Flush(); + } + } + /// /// Converts any stream into a sparse stream. /// @@ -483,6 +510,16 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + CheckDisposed(); + + if (CanWrite) + { + FlushWrappedStream(_wrapped, flushToDisk); + } + } + public override int Read(byte[] buffer, int offset, int count) { CheckDisposed(); @@ -688,6 +725,20 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + _sync.Wait(); + + try + { + content.Flush(flushToDisk); + } + finally + { + _sync.Release(); + } + } + public override async Task FlushAsync(CancellationToken cancellationToken) { await _sync.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -1043,4 +1094,4 @@ public override async ValueTask DisposeAsync() public override string ToString() => $"Syncrhonized[{content}]"; } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/StripedStream.cs b/Library/DiscUtils.Streams/StripedStream.cs index 1eff64483..6a9971256 100644 --- a/Library/DiscUtils.Streams/StripedStream.cs +++ b/Library/DiscUtils.Streams/StripedStream.cs @@ -116,6 +116,14 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + foreach (var stream in _wrapped) + { + stream.Flush(flushToDisk); + } + } + public override int Read(byte[] buffer, int offset, int count) { if (!CanRead) @@ -357,4 +365,4 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/SubStream.cs b/Library/DiscUtils.Streams/SubStream.cs index 4e9dda86b..91abaadde 100644 --- a/Library/DiscUtils.Streams/SubStream.cs +++ b/Library/DiscUtils.Streams/SubStream.cs @@ -109,6 +109,11 @@ public override void Flush() Parent.Flush(); } + public override void Flush(bool flushToDisk) + { + FlushWrappedStream(Parent, flushToDisk); + } + public override int Read(byte[] buffer, int offset, int count) { if (count < 0) @@ -285,4 +290,4 @@ private IEnumerable OffsetExtents(IEnumerable src) yield return new StreamExtent(e.Start - _first, e.Length); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/ThreadSafeStream.cs b/Library/DiscUtils.Streams/ThreadSafeStream.cs index 3dc301183..30d4951f6 100644 --- a/Library/DiscUtils.Streams/ThreadSafeStream.cs +++ b/Library/DiscUtils.Streams/ThreadSafeStream.cs @@ -239,6 +239,14 @@ public override void Flush() } } + public override void Flush(bool flushToDisk) + { + lock (_common) + { + Wrapped.Flush(flushToDisk); + } + } + /// /// Reads data from the stream. /// @@ -398,4 +406,4 @@ public void Dispose() #endregion } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/WrappingMappedStream.cs b/Library/DiscUtils.Streams/WrappingMappedStream.cs index 09f52e89e..fd481adc1 100644 --- a/Library/DiscUtils.Streams/WrappingMappedStream.cs +++ b/Library/DiscUtils.Streams/WrappingMappedStream.cs @@ -114,6 +114,11 @@ public override void Flush() WrappedStream.Flush(); } + public override void Flush(bool flushToDisk) + { + FlushWrappedStream(WrappedStream, flushToDisk); + } + public override Task FlushAsync(CancellationToken cancellationToken) => WrappedStream.FlushAsync(cancellationToken); @@ -205,4 +210,4 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Streams/WrappingStream.cs b/Library/DiscUtils.Streams/WrappingStream.cs index 7104f8599..ba13b7092 100644 --- a/Library/DiscUtils.Streams/WrappingStream.cs +++ b/Library/DiscUtils.Streams/WrappingStream.cs @@ -77,6 +77,11 @@ public override void Flush() _wrapped.Flush(); } + public override void Flush(bool flushToDisk) + { + _wrapped.Flush(flushToDisk); + } + public override int Read(byte[] buffer, int offset, int count) { return _wrapped.Read(buffer, offset, count); @@ -161,4 +166,4 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Vhd/DynamicStream.cs b/Library/DiscUtils.Vhd/DynamicStream.cs index c6b3017a5..8c42bcb9b 100644 --- a/Library/DiscUtils.Vhd/DynamicStream.cs +++ b/Library/DiscUtils.Vhd/DynamicStream.cs @@ -150,6 +150,19 @@ public override long Position public override void Flush() { CheckDisposed(); + if (CanWrite) + { + _fileStream.Flush(); + } + } + + public override void Flush(bool flushToDisk) + { + CheckDisposed(); + if (CanWrite) + { + FlushWrappedStream(_fileStream, flushToDisk); + } } public override IEnumerable MapContent(long start, long length) @@ -1236,4 +1249,4 @@ private async ValueTask UpdateFooterAsync(CancellationToken cancellationToken) await _fileStream.WriteAsync(_footerCache, cancellationToken).ConfigureAwait(false); } } -} \ No newline at end of file +} diff --git a/Library/DiscUtils.Vhdx/ContentStream.cs b/Library/DiscUtils.Vhdx/ContentStream.cs index f2bd91043..71760c3fc 100644 --- a/Library/DiscUtils.Vhdx/ContentStream.cs +++ b/Library/DiscUtils.Vhdx/ContentStream.cs @@ -116,6 +116,18 @@ public override void Flush() _fileStream.Flush(); } + public override void Flush(bool flushToDisk) + { + CheckDisposed(); + + if (!CanWrite) + { + throw new InvalidOperationException("Attempt to flush to read-only VHDX"); + } + + _fileStream.Flush(flushToDisk); + } + public override IEnumerable MapContent(long start, long length) { CheckDisposed(); @@ -650,4 +662,4 @@ private void CheckDisposed() } #endif } -} \ No newline at end of file +} diff --git a/README.md b/README.md index 5a16ba211..01e2620d8 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,13 @@ Filesystem constructors normally expect a stream beginning at the filesystem, su | [FAT12 / FAT16 / FAT32](Library/DiscUtils.Fat/README.md) | Yes | Yes | Yes | Includes floppy and partition formatting | | [HFS+](Library/DiscUtils.HfsPlus/README.md) | Yes | No | No | Unix metadata and allocation extents; not APFS | | [ISO 9660 / Joliet](Library/DiscUtils.Iso9660/README.md) | Yes | Yes | No | CDReader also supports Rock Ridge; CDBuilder builds new images, including bootable-image scenarios | -| [NTFS](Library/DiscUtils.Ntfs/README.md) | Yes | Yes | Yes | Includes alternate data streams, security descriptors, reparse points and allocation metadata | +| [NTFS](Library/DiscUtils.Ntfs/README.md) | Yes | Yes | Yes | Alternate data streams, security metadata and guarded `$LogFile` journal recovery | | [SquashFS](Library/DiscUtils.SquashFs/README.md) | Yes | Yes | No | Separate reader and builder; zlib included, other codecs require callbacks | | [UDF](Library/DiscUtils.Udf/README.md) | Yes | UDF 2.01 | No | Virtual image builder; size/used/free-space reader properties are not implemented | | [XFS](Library/DiscUtils.Xfs/README.md) | Yes | No | No | Unix metadata and allocation extents | +NTFS also supports journal analysis and selective recovery after an unclean shutdown. Writable filesystem opens refuse pending recovery by default; read-only opens can replay supported changes in memory. Persistent replay requires explicit opt-in, an offline and exclusively owned volume, and a backing stream with suitable durability guarantees; metadata undo cannot yet be persisted. See the [NTFS package README](Library/DiscUtils.Ntfs/README.md) and [journal durability notes](docs/ntfs-journal-replay-durability.md). + ### Archive filesystem views These adapters expose archive entries through the common filesystem API. diff --git a/Tests/LibraryTests/Iscsi/ScsiSynchronizeCacheTest.cs b/Tests/LibraryTests/Iscsi/ScsiSynchronizeCacheTest.cs new file mode 100644 index 000000000..240a495c5 --- /dev/null +++ b/Tests/LibraryTests/Iscsi/ScsiSynchronizeCacheTest.cs @@ -0,0 +1,58 @@ +// +// Copyright (c) 2026, DiscUtils contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +using System; +using DiscUtils.Iscsi; + +namespace LibraryTests.Iscsi; + +public class ScsiSynchronizeCacheTest +{ + [Fact] + public void SynchronizeCache10RequestsWholeLunAndWaitsForCompletion() + { + var command = new ScsiSynchronizeCache10Command(0x123456); + var bytes = new byte[command.Size]; + for (var i = 0; i < bytes.Length; i++) + { + bytes[i] = 0xFF; + } + + command.WriteTo(bytes); + + Assert.Equal(10, command.Size); + Assert.Equal((ulong)0x123456, command.TargetLun); + Assert.Equal(TaskAttributes.Simple, command.TaskAttributes); + // LBA 0, number of blocks 0: all blocks through the end of the LUN. + // IMMED=0: do not acknowledge before synchronization finishes. + Assert.Equal(new byte[] { 0x35, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, bytes); + } + + [Fact] + public void DiskStreamOverridesTheStrongSparseFlushOverload() + { + var flush = typeof(DiskStream).GetMethod(nameof(DiskStream.Flush), new[] { typeof(bool) }); + + Assert.NotNull(flush); + Assert.Equal(typeof(DiskStream), flush.DeclaringType); + } +} diff --git a/Tests/LibraryTests/Ntfs/Data/journal-s4kn.ntfs.gz b/Tests/LibraryTests/Ntfs/Data/journal-s4kn.ntfs.gz new file mode 100644 index 000000000..bbc0a2cbf Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-s4kn.ntfs.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-s4kn.windows.txt.gz b/Tests/LibraryTests/Ntfs/Data/journal-s4kn.windows.txt.gz new file mode 100644 index 000000000..025ee9908 Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-s4kn.windows.txt.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-s512.ntfs.gz b/Tests/LibraryTests/Ntfs/Data/journal-s512.ntfs.gz new file mode 100644 index 000000000..4a07dd302 Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-s512.ntfs.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-s512.windows.txt.gz b/Tests/LibraryTests/Ntfs/Data/journal-s512.windows.txt.gz new file mode 100644 index 000000000..e654111cf Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-s512.windows.txt.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-s64k.ntfs.gz b/Tests/LibraryTests/Ntfs/Data/journal-s64k.ntfs.gz new file mode 100644 index 000000000..d061aa976 Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-s64k.ntfs.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-s64k.windows.txt.gz b/Tests/LibraryTests/Ntfs/Data/journal-s64k.windows.txt.gz new file mode 100644 index 000000000..9978c872b Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-s64k.windows.txt.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-x4k-a.ntfs.gz b/Tests/LibraryTests/Ntfs/Data/journal-x4k-a.ntfs.gz new file mode 100644 index 000000000..b519cd5ae Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-x4k-a.ntfs.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-x4k-a.windows.txt.gz b/Tests/LibraryTests/Ntfs/Data/journal-x4k-a.windows.txt.gz new file mode 100644 index 000000000..5882ca757 Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-x4k-a.windows.txt.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-x4k-c.ntfs.gz b/Tests/LibraryTests/Ntfs/Data/journal-x4k-c.ntfs.gz new file mode 100644 index 000000000..badd65bd1 Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-x4k-c.ntfs.gz differ diff --git a/Tests/LibraryTests/Ntfs/Data/journal-x4k-c.windows.txt.gz b/Tests/LibraryTests/Ntfs/Data/journal-x4k-c.windows.txt.gz new file mode 100644 index 000000000..d3dffa455 Binary files /dev/null and b/Tests/LibraryTests/Ntfs/Data/journal-x4k-c.windows.txt.gz differ diff --git a/Tests/LibraryTests/Ntfs/Journal/JournalScenario.cs b/Tests/LibraryTests/Ntfs/Journal/JournalScenario.cs new file mode 100644 index 000000000..6e1eb536f --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/JournalScenario.cs @@ -0,0 +1,262 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DiscUtils; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Builds a volume that "Windows crashed on" from two DiscUtils images: and +/// a set of changes. Every MFT record, index buffer and data cluster that differs becomes a log record (redo: the +/// after bytes, undo: the before bytes) in one transaction, the way NTFS logs page images. +/// +internal sealed class JournalScenario +{ + public const ushort MftData = 0x18; + public const ushort IndexAllocation = 0x40; + public const ushort PlainData = 0x68; + public const uint Transaction = 0x40; + + private JournalScenario(byte[] before, byte[] after, int clusterSize) + { + Before = before; + After = after; + ClusterSize = clusterSize; + } + + public byte[] Before { get; } + + public byte[] After { get; } + + public int ClusterSize { get; } + + /// Clusters of $LogFile, which the journal itself occupies (not compared). + public HashSet LogClusters { get; } = []; + + public List Records { get; } = []; + + /// + /// Formats a volume, runs on it with DiscUtils and turns the difference into log records. + /// + public static JournalScenario Create(Action change, int sizeMb = 16, Action? setup = null) + { + var volume = new MemoryStream(); + var geometry = Geometry.FromCapacity(sizeMb * 1024L * 1024); + using (var ntfs = NtfsFileSystem.Format(volume, "JOURNAL", geometry, 0, geometry.TotalSectorsLong)) + { + setup?.Invoke(ntfs); + } + + var before = volume.ToArray(); + using (var ntfs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore)) + { + change(ntfs); + } + + var after = volume.ToArray(); + var scenario = new JournalScenario(before, after, 4096); + scenario.Diff(); + return scenario; + } + + /// Gets a writable copy of an image. + public static MemoryStream Copy(byte[] image) + { + var copy = new MemoryStream(); + copy.Write(image, 0, image.Length); + copy.Position = 0; + return copy; + } + + /// + /// Writes the journal for the scenario into : a checkpoint, the records (each chained + /// to the previous one of the transaction for undo), and, when , the end of the + /// transaction. Returns the writer (for the LSNs). + /// + public TestJournalWriter WriteJournal(Stream volume, bool committed, int majorVersion = 2, long? startOffset = null, + Action? beforeRecords = null, Func? adjust = null) + { + var writer = new TestJournalWriter(volume, majorVersion, startOffset); + var restart = writer.Checkpoint( + [ + (MftData, 0x80, 0x0001000000000000UL, 0, ""), + (IndexAllocation, 0xA0, 0x0005000000000005UL, 4096, "$I30"), + (PlainData, 0x80, 0x0001000000000006UL, 0, ""), + ]); + beforeRecords?.Invoke(writer); + ulong previous = 0; + foreach (var record in Records) + { + var r = adjust != null ? adjust(record) : record; + r.UndoNextLsn = previous; + previous = writer.Append(r); + } + + if (committed) + { + writer.Append(new TestLogRecord + { + Redo = NtfsLogOperation.ForgetTransaction, + Undo = NtfsLogOperation.CompensationLogRecord, + TransactionId = Transaction, + }); + } + + writer.Finish(restart); + return writer; + } + + /// + /// Asserts that equals outside $LogFile, ignoring what a + /// replay legitimately changes in file records and index buffers: the LSN and the update sequence number. + /// + public void AssertVolumeEquals(byte[] expected, Stream volume) + { + Assert.Equal(expected.Length, volume.Length); + var actual = new byte[expected.Length]; + JournalUtilities.ReadExactly(volume, 0, actual, 0, actual.Length); + for (long lcn = 0; lcn < expected.Length / ClusterSize; lcn++) + { + if (LogClusters.Contains(lcn)) + { + continue; + } + + var offset = (int)(lcn * ClusterSize); + var e = Normalize(expected, offset); + var a = Normalize(actual, offset); + if (!e.AsSpan().SequenceEqual(a)) + { + var first = 0; + while (e[first] == a[first]) + { + first++; + } + + Assert.Fail($"Cluster {lcn} differs at 0x{first:X}: expected {e[first]:X2}, got {a[first]:X2}"); + } + } + } + + private byte[] Normalize(byte[] image, int offset) + { + var cluster = image.AsSpan(offset, ClusterSize).ToArray(); + var unit = EndianUtilities.ToUInt32LittleEndian(cluster, 0) == JournalUtilities.SignatureIndx ? ClusterSize : 1024; + for (var start = 0; start + unit <= ClusterSize; start += unit) + { + var signature = EndianUtilities.ToUInt32LittleEndian(cluster, start); + if (signature == JournalUtilities.SignatureFile || signature == JournalUtilities.SignatureIndx) + { + JournalUtilities.RemoveFixups(cluster, start, unit); + var usaOffset = EndianUtilities.ToUInt16LittleEndian(cluster, start + 4); + var usaCount = EndianUtilities.ToUInt16LittleEndian(cluster, start + 6); + Array.Clear(cluster, start + 8, 8); + Array.Clear(cluster, start + usaOffset, 2 * usaCount); + } + } + + return cluster; + } + + private void Diff() + { + using var afterVolume = Copy(After); + using var ntfs = new NtfsFileSystem(afterVolume, NtfsJournalMode.Ignore); + foreach (var range in ntfs.PathToClusters("$LogFile")) + { + for (var lcn = range.Offset; lcn < range.Offset + range.Count; lcn++) + { + LogClusters.Add(lcn); + } + } + + var mftVcn = new Dictionary(); + long vcn = 0; + foreach (var range in ntfs.PathToClusters("$MFT")) + { + for (var lcn = range.Offset; lcn < range.Offset + range.Count; lcn++) + { + mftVcn[lcn] = vcn++; + } + } + + for (long lcn = 0; lcn < After.Length / ClusterSize; lcn++) + { + var offset = (int)(lcn * ClusterSize); + if (LogClusters.Contains(lcn) || Before.AsSpan(offset, ClusterSize).SequenceEqual(After.AsSpan(offset, ClusterSize))) + { + continue; + } + + if (mftVcn.TryGetValue(lcn, out var recordVcn)) + { + for (var r = 0; r < ClusterSize / 1024; r++) + { + var start = offset + (r * 1024); + if (Before.AsSpan(start, 1024).SequenceEqual(After.AsSpan(start, 1024))) + { + continue; + } + + Records.Add(new TestLogRecord + { + Redo = NtfsLogOperation.InitializeFileRecordSegment, + Undo = NtfsLogOperation.InitializeFileRecordSegment, + TransactionId = Transaction, + RedoData = Logical(After, start, 1024), + UndoData = Logical(Before, start, 1024), + TargetAttribute = MftData, + TargetVcn = recordVcn, + ClusterBlockOffset = r * 2, + Lcns = [lcn], + }); + } + } + else if (EndianUtilities.ToUInt32LittleEndian(After, offset) == JournalUtilities.SignatureIndx) + { + Records.Add(new TestLogRecord + { + Redo = NtfsLogOperation.UpdateNonresidentValue, + Undo = NtfsLogOperation.UpdateNonresidentValue, + TransactionId = Transaction, + RedoData = Logical(After, offset, ClusterSize), + UndoData = Logical(Before, offset, ClusterSize), + TargetAttribute = IndexAllocation, + TargetVcn = lcn, + Lcns = [lcn], + }); + } + else + { + Records.Add(new TestLogRecord + { + Redo = NtfsLogOperation.UpdateNonresidentValue, + Undo = NtfsLogOperation.UpdateNonresidentValue, + TransactionId = Transaction, + RedoData = After.AsSpan(offset, ClusterSize).ToArray(), + UndoData = Before.AsSpan(offset, ClusterSize).ToArray(), + TargetAttribute = PlainData, + TargetVcn = lcn, + Lcns = [lcn], + }); + } + } + } + + /// The bytes of a file record or index buffer with the update sequence array removed. + private static byte[] Logical(byte[] image, int offset, int length) + { + var bytes = image.AsSpan(offset, length).ToArray(); + var signature = EndianUtilities.ToUInt32LittleEndian(bytes, 0); + if (signature == JournalUtilities.SignatureFile || signature == JournalUtilities.SignatureIndx) + { + Assert.True(JournalUtilities.RemoveFixups(bytes, 0, length)); + } + + return bytes; + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalClientIdentityTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalClientIdentityTest.cs new file mode 100644 index 000000000..061f4ef9d --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalClientIdentityTest.cs @@ -0,0 +1,314 @@ +using System; +using System.IO; +using System.Text; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace LibraryTests.Ntfs.Journal; + +public partial class NtfsJournalEdgeCaseTest +{ + private const int RestartArea = 0x30; + private const int FirstClient = RestartArea + 0x40; + private const int SecondClient = FirstClient + 0xA0; + private const ushort ActiveClientSequence = 7; + + [Fact] + public void AClientRecordWithBothIdentityFieldsMatchingIsReplayed() + { + using var volume = Volume(); + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume) { ClientSequenceNumber = ActiveClientSequence }; + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(target, Pattern)); + writer.Append(Forget()); + writer.Finish(restart); + + var info = NtfsJournal.Replay(volume); + Assert.True(info.Replayed); + Assert.Equal(1, info.RecordsToRedo); + Assert.Equal(Pattern, Read(volume, target * writer.Layout.BytesPerCluster, Pattern.Length)); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AnUnrelatedForgetCannotFinishAnNtfsTransaction(bool anotherSlot) + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume) { ClientSequenceNumber = ActiveClientSequence }; + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(FreeCluster(volume), Pattern)); + AppendUnrelated(writer, Forget(), anotherSlot); + writer.Finish(restart); + if (anotherSlot) + { + AddFreeClientSlot(writer); + } + + var info = NtfsJournal.Analyze(volume); + Assert.Equal(NtfsJournalStatus.NothingToReplay, info.Status); + Assert.Equal(1, info.RecordsToUndo); + Assert.Equal(1, info.TransactionsToUndo); + Assert.False(info.CanReplayPersistently); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AnUnrelatedRecordCannotFreeAnNtfsDirtyPage(bool anotherSlot) + { + using var volume = Volume(); + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume) { ClientSequenceNumber = ActiveClientSequence }; + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(target, Pattern)); + writer.Append(Forget()); + var last = AppendUnrelated(writer, DeleteClusters(target), anotherSlot); + writer.Finish(restart); + if (anotherSlot) + { + AddFreeClientSlot(writer); + } + + var info = NtfsJournal.ReplayInMemory(volume, out var overlay); + using var recovered = overlay; + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.Equal(last, info.LastLsn); + Assert.Equal(Pattern, Read(recovered, target * writer.Layout.BytesPerCluster, Pattern.Length)); + + Assert.Equal(!anotherSlot, info.CanReplayPersistently); + if (anotherSlot) + { + AssertMultiSlotResetRefused(volume); + } + else + { + Assert.True(NtfsJournal.Replay(volume).Replayed); + Assert.Equal(Pattern, Read(volume, target * writer.Layout.BytesPerCluster, Pattern.Length)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RedoSkipsUnrelatedRecordsBeforeTheCheckpoint(bool anotherSlot) + { + using var volume = Volume(); + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume) { ClientSequenceNumber = ActiveClientSequence }; + var first = writer.Append(Raw(target, Pattern)); + AppendUnrelated(writer, Raw(target, [0xA5]), anotherSlot); + AppendUnrelated(writer, new TestLogRecord { Redo = (NtfsLogOperation)0xFFFF }, anotherSlot); + var restart = writer.Checkpoint(Attributes, dirtyPages: [(Data, target, first, [target])]); + writer.Finish(restart); + if (anotherSlot) + { + AddFreeClientSlot(writer); + } + + var info = NtfsJournal.ReplayInMemory(volume, out var overlay); + using var recovered = overlay; + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.Equal(1, info.RecordsToRedo); + Assert.Equal(Pattern, Read(recovered, target * writer.Layout.BytesPerCluster, Pattern.Length)); + Assert.Equal(new byte[Pattern.Length], Read(volume, target * writer.Layout.BytesPerCluster, Pattern.Length)); + } + + [Theory] + [InlineData(-1, "NTFS restart record")] + [InlineData(0x08, "checkpoint record")] + [InlineData(0x10, "open attribute table dump")] + [InlineData(0x18, "attribute names dump")] + [InlineData(0x20, "dirty page table dump")] + [InlineData(0x28, "transaction table dump")] + public void CheckpointReferencesCannotNameAnotherClient(int restartField, string role) + { + using var volume = Volume(); + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume); + var first = writer.Append(Raw(target, Pattern)); + var restart = writer.Checkpoint(Attributes, dirtyPages: [(Data, target, first, [target])], + transactions: [(Tx, TransactionState.Committed, first, first)]); + writer.Finish(restart); + AddFreeClientSlot(writer); + var log = LfsLog.Open(writer.Storage, out _); + var lsn = restartField < 0 ? restart : U64(log.ReadRecord(restart).ClientData, restartField); + Assert.NotEqual(0UL, lsn); + PatchJournalRecord(writer, lsn, (page, offset) => Put16(page, offset + 0x1E, 1)); + + AssertReplayRefused(volume, $"{role} at {lsn:X} does not belong to the active NTFS client"); + } + + [Theory] + [InlineData(false, "not in the log")] + [InlineData(true, "wrong LFS record type")] + public void ADeclaredTableDumpMustReferenceAnLfsClientRecord(bool wrongType, string reason) + { + using var volume = WithRestart((writer, checkpoint) => + { + var dump = new TestLogRecord { Redo = NtfsLogOperation.OpenAttributeTableDump, RedoData = Table(0x28, 1) }; + var lsn = wrongType ? writer.AppendRaw(dump.ClientData(), LfsRecord.TypeClientRestart, 0, 0) : 0; + return RestartRecord(1, checkpoint, oat: lsn, oatLength: dump.RedoData.Length); + }); + + AssertReplayRefused(volume, reason); + } + + [Theory] + [InlineData("dirty page")] + [InlineData("first record")] + [InlineData("previous record")] + public void CheckpointHistoryCannotNameAnObsoleteClient(string reference) + { + using var volume = Volume(); + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume) { ClientSequenceNumber = ActiveClientSequence }; + var first = writer.Append(Raw(target, Pattern)); + var obsolete = AppendUnrelated(writer, Raw(target, [0xA5]), anotherSlot: false); + var restart = writer.Checkpoint(Attributes, + dirtyPages: [(Data, target, first, [target]), (Data, target + 1, reference == "dirty page" ? obsolete : 0, [target + 1])], + transactions: [(Tx, TransactionState.Committed, reference == "first record" ? obsolete : first, + reference == "previous record" ? obsolete : first)]); + writer.Finish(restart); + + AssertReplayRefused(volume, "does not belong to the active NTFS client"); + } + + [Fact] + public void ALogRecordCannotNameAClientIndexOutsideTheRestartArray() + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(FreeCluster(volume), Pattern)); + writer.Append(Forget()); + AppendUnrelated(writer, Forget(), anotherSlot: true); + writer.Finish(restart); + + AssertReplayRefused(volume, "client index 1 outside the restart client array"); + } + + [Theory] + [InlineData(FirstClient + 0x12, 0)] + [InlineData(FirstClient + 0x12, 2)] + [InlineData(FirstClient + 0x10, 0)] + [InlineData(RestartArea + 0x0A, 0)] + public void MalformedRestartClientListsAreRefused(int offset, int value) + { + using var volume = WithRecords(); + PatchRestartPages(new TestJournalWriter(volume), page => Put16(page, offset, (ushort)value)); + + AssertReplayRefused(volume, "invalid LFS client list"); + } + + [Theory] + [InlineData("Other", "no NTFS client in use")] + [InlineData("NTFS", "more than one NTFS client in use")] + public void TheInUseListMustIdentifyExactlyOneNtfsClient(string name, string reason) + { + using var volume = WithRecords(); + PatchRestartPages(new TestJournalWriter(volume), page => + { + if (name == "NTFS") + { + PutOtherClientAtHead(page, name); + } + else + { + PutClientName(page, FirstClient, name); + } + }); + + AssertReplayRefused(volume, reason); + } + + [Fact] + public void AnActiveNtfsClientCanFollowAnUnrelatedClientInTheInUseList() + { + using var volume = Volume(); + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(target, Pattern)); + writer.Append(Forget()); + writer.AppendRaw([0x01], LfsRecord.TypeClientRecord, Tx, 0, clientIndex: 1); + writer.Finish(restart); + PatchRestartPages(writer, page => PutOtherClientAtHead(page, "Other")); + + var info = NtfsJournal.ReplayInMemory(volume, out var overlay); + using var recovered = overlay; + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.False(info.CanReplayPersistently); + Assert.Equal(Pattern, Read(recovered, target * writer.Layout.BytesPerCluster, Pattern.Length)); + + AssertMultiSlotResetRefused(volume); + } + + [Fact] + public void EqualLsnRestartPagesMustAgreeOnTheActiveNtfsClientSequence() + { + using var volume = WithRecords(); + var writer = new TestJournalWriter(volume); + PatchRestartPages(writer, page => PutOtherClientAtHead(page, "Other")); + var second = new byte[TestJournalWriter.PageSize]; + writer.Storage.Read(TestJournalWriter.PageSize, second, 0, second.Length); + Assert.True(RemoveFixups(second, 0, second.Length)); + Put16(second, FirstClient + 0x14, 1); + ApplyFixups(second, 0, second.Length); + writer.Storage.Write(TestJournalWriter.PageSize, second, 0, second.Length); + + AssertReplayRefused(volume, "disagree about the checkpoint at the same LSN"); + } + + private static ulong AppendUnrelated(TestJournalWriter writer, TestLogRecord record, bool anotherSlot) => + writer.AppendRaw(record.ClientData(), LfsRecord.TypeClientRecord, record.TransactionId, record.UndoNextLsn, + clientIndex: (ushort)(anotherSlot ? 1 : 0), + clientSequence: anotherSlot ? writer.ClientSequenceNumber : (ushort)(writer.ClientSequenceNumber - 1)); + + private static void AddFreeClientSlot(TestJournalWriter writer) => PatchRestartPages(writer, page => + { + Put16(page, RestartArea + 0x08, 2); + Put16(page, RestartArea + 0x0A, 1); + Put16(page, SecondClient + 0x10, LfsRestartPage.NoClient); + Put16(page, SecondClient + 0x12, LfsRestartPage.NoClient); + }); + + private static void PutOtherClientAtHead(byte[] page, string name) + { + Put16(page, RestartArea + 0x08, 2); + Put16(page, RestartArea + 0x0C, 1); + Put16(page, FirstClient + 0x10, 1); + Put16(page, SecondClient + 0x10, LfsRestartPage.NoClient); + Put16(page, SecondClient + 0x12, 0); + PutClientName(page, SecondClient, name); + } + + private static void PutClientName(byte[] page, int offset, string name) + { + var bytes = Encoding.Unicode.GetBytes(name); + Put32(page, offset + 0x1C, (uint)bytes.Length); + bytes.CopyTo(page, offset + 0x20); + } + + private static void AssertMultiSlotResetRefused(MemoryStream volume) + { + var before = volume.ToArray(); + Assert.Contains("single LFS client slot", Assert.Throws(() => NtfsJournal.Replay(volume)).Message); + Assert.True(before.AsSpan().SequenceEqual(volume.ToArray())); + } + + private static void PatchJournalRecord(TestJournalWriter writer, ulong lsn, Action patch) + { + var offset = writer.LsnToOffsetForTest(lsn); + var inPage = (int)(offset % TestJournalWriter.PageSize); + var page = new byte[TestJournalWriter.PageSize]; + writer.Storage.Read(offset - inPage, page, 0, page.Length); + Assert.True(RemoveFixups(page, 0, page.Length)); + patch(page, inPage); + ApplyFixups(page, 0, page.Length); + writer.Storage.Write(offset - inPage, page, 0, page.Length); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalDamagedLogTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalDamagedLogTest.cs new file mode 100644 index 000000000..6963091fd --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalDamagedLogTest.cs @@ -0,0 +1,244 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Logs that are unusual or damaged in one specific place: restart pages, records that are no NTFS records, the +/// checkpoint at the very end, hot fixes, and pages the log describes in ways DiscUtils must not trust. +/// +public partial class NtfsJournalEdgeCaseTest +{ + private static readonly byte[] Pattern = Enumerable.Repeat((byte)0x3C, 16).ToArray(); + + private static TestLogRecord HotFix(long vcn, long[] lcns) => + new() { Redo = NtfsLogOperation.HotFix, Undo = NtfsLogOperation.Noop, TransactionId = Tx, TargetAttribute = Data, TargetVcn = vcn, Lcns = lcns }; + + private static TestLogRecord Noop() => + new() { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0x18 }; + + [Fact] + public void ALogWithNoClientInUse_IsClean() + { + using var probe = Volume(); + using var volume = WithRecords(Raw(FreeCluster(probe), Pattern), Forget()); + PatchRestartPages(new TestJournalWriter(volume), page => EndianUtilities.WriteBytesLittleEndian((ushort)0xFFFF, page, 0x30 + 0x0C)); + + var info = NtfsJournal.Analyze(volume); + Assert.Equal(NtfsJournalStatus.Clean, info.Status); + Assert.Contains("cleanly", info.Detail); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AChkdskMarkOnTheSecondRestartPageOnly_StillMeansChkdskRan(bool firstPageDamaged) + { + using var probe = Volume(); + using var volume = WithRecords(Raw(FreeCluster(probe), Pattern), Forget()); + var w = new TestJournalWriter(volume); + w.Storage.Write(TestJournalWriter.PageSize, BitConverter.GetBytes(JournalUtilities.SignatureChkd), 0, 4); + if (firstPageDamaged) + { + w.Storage.Write(0, [0x58, 0x58, 0x58, 0x58], 0, 4); + } + + var info = NtfsJournal.Analyze(volume); + Assert.Equal(NtfsJournalStatus.Clean, info.Status); + Assert.Contains("chkdsk", info.Detail); + } + + [Fact] + public void ADamagedSecondRestartPage_LeavesTheFirst() + { + using var probe = Volume(); + var x = FreeCluster(probe); + using var volume = WithRecords(Raw(x, Pattern), Forget()); + new TestJournalWriter(volume).Storage.Write(TestJournalWriter.PageSize, [0x58, 0x58, 0x58, 0x58], 0, 4); + + NtfsJournal.Replay(volume); + Assert.Equal(0x3C, Read(volume, x * 4096, 1)[0]); + } + + [Fact] + public void ARecordThatIsNoNtfsRecord_IsRefused_AfterTheCheckpoint_WhereRedoStarts_AndInAnUndoChain() + { + // After the checkpoint: the analysis reads it. + using (var volume = Volume()) + { + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint(Attributes); + w.AppendRaw(new byte[8], LfsRecord.TypeClientRecord, Tx, 0); + w.Finish(restart); + AssertRefused(volume, "too short"); + } + + // Before it, where the redo pass starts: a page has been dirty since then. + using (var volume = Volume()) + { + var x = FreeCluster(volume); + var w = new TestJournalWriter(volume); + var bad = w.AppendRaw(new byte[8], LfsRecord.TypeClientRecord, Tx, 0); + var restart = w.Checkpoint(Attributes, dirtyPages: [(Data, x, bad, [x])]); + w.Finish(restart); + AssertRefused(volume, "too short"); + } + + // Before it, in the undo chain of a transaction the crash interrupted. + using (var volume = Volume()) + { + var w = new TestJournalWriter(volume); + var bad = w.AppendRaw(new byte[8], LfsRecord.TypeClientRecord, 0x90, 0); + var restart = w.Checkpoint(Attributes, transactions: [(0x90, TransactionState.Active, bad, bad)]); + w.Finish(restart); + AssertRefused(volume, "too short"); + } + } + + [Theory] + [InlineData(0x04, 4)] + [InlineData(0x08, 4)] + [InlineData(0x08, -0x20)] + public void ARecordWithMisplacedRedoOrUndoData_IsNoRecord(int field, int shift) + { + var record = new TestLogRecord + { + Redo = NtfsLogOperation.UpdateResidentValue, Undo = NtfsLogOperation.UpdateResidentValue, RedoData = new byte[8], UndoData = new byte[8], Lcns = [1], + }; + var data = record.ClientData(); + EndianUtilities.WriteBytesLittleEndian((ushort)(EndianUtilities.ToUInt16LittleEndian(data, field) + shift), data, field); + + Assert.Null(NtfsLogRecord.TryParse(new LfsRecord { Lsn = 1, ClientData = data }, out var error)); + Assert.Contains("misplaced", error); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ACheckpointThatIsTheLastRecord_StartsRedoAtTheOldestDirtyPage_OrTransaction(bool dirtyPages) + { + using var probe = Volume(); + var x = FreeCluster(probe); + + // The checkpoint LSN is written before the record it names, so the log is built twice: first to learn + // where the record after the restart record lands. + ulong Build(MemoryStream volume, ulong checkpoint, out ulong change) + { + var w = new TestJournalWriter(volume); + change = w.Append(Raw(x, Pattern)); + var pages = new List<(uint, long, ulong, long[])>(); + var transactions = new List<(uint, TransactionState, ulong, ulong)>(); + if (dirtyPages) + { + pages.Add((Data, x, change, [x])); + pages.Add((Data, x + 1, 0, [x + 1])); + } + else + { + transactions.Add((Tx, TransactionState.Committed, change, change)); + transactions.Add((0x68, TransactionState.Committed, 0, 0)); + } + + var restart = w.Checkpoint(Attributes, pages, transactions, checkpointLsn: checkpoint); + var last = w.Append(Noop()); + w.Finish(restart); + return last; + } + + using var dry = Volume(); + var lastLsn = Build(dry, 0, out _); + using var volume = Volume(); + Assert.Equal(lastLsn, Build(volume, lastLsn, out var change)); + + var layout = JournalVolumeLayout.Read(volume); + var log = LfsLog.Open(LogFileStorage.Open(volume, layout), out _); + var analysis = NtfsLogAnalysis.Run(log, layout); + Assert.Equal(0, analysis.RecordsAfterCheckpoint); + Assert.Equal(change, analysis.RedoLsn); + + // Redo needs a dirty page; a committed transaction alone leaves nothing to do. + var info = NtfsJournal.Replay(volume); + Assert.Equal(dirtyPages ? NtfsJournalStatus.NeedsReplay : NtfsJournalStatus.NothingToReplay, info.Status); + Assert.Equal(dirtyPages ? 0x3C : 0, Read(volume, x * 4096, 1)[0]); + } + + [Fact] + public void HotFixes_OfACleanPage_WithoutACluster_OrOfAFreedCluster_MoveNothing() + { + using var volume = Volume(); + var x = FreeCluster(volume); + var y = FreeCluster(volume, 1); + var z = FreeCluster(volume, 2); + using var result = WithRecordsOn(volume, + HotFix(9000, [y]), + Raw(z, Pattern), + HotFix(z, []), + Raw(x, Pattern), + DeleteClusters(x), + HotFix(x, [y]), + Forget()); + + var info = NtfsJournal.Replay(result); + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.Equal(0x3C, Read(result, z * 4096, 1)[0]); + Assert.Equal(0, Read(result, x * 4096, 1)[0]); + Assert.Equal(0, Read(result, y * 4096, 1)[0]); + } + + [Theory] + [InlineData(-5L)] + [InlineData(1L << 40)] + public void ARecordNamingAClusterOffTheVolume_IsRefused(long lcn) + { + using var volume = WithRecords(Op(NtfsLogOperation.UpdateNonresidentValue, new byte[8], Data, 3, [lcn]), Forget()); + AssertRefused(volume, "not on the volume"); + } + + [Fact] + public void AnIndexBufferImageWithoutAFixupArray_IsWrittenAsLogged() + { + using var volume = Volume(); + var x = FreeCluster(volume); + var image = new byte[4096]; + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureIndx, image, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)0x28, image, 4); + image[0x100] = 0x77; + using var result = WithRecordsOn(volume, Op(NtfsLogOperation.UpdateNonresidentValue, image, Index, 0, [x]), Forget()); + + NtfsJournal.Replay(result); + Assert.True(Read(result, x * 4096, 4096).SequenceEqual(image)); + } + + [Fact] + public void PartOfAnIndexBufferWrittenIntoAnEmptyCluster_IsWrittenAsLogged() + { + using var volume = Volume(); + var x = FreeCluster(volume); + using var result = WithRecordsOn(volume, Op(NtfsLogOperation.UpdateNonresidentValue, Pattern, Index, 0, [x], recordOffset: 0x100), Forget()); + + NtfsJournal.Replay(result); + var cluster = Read(result, x * 4096, 4096); + Assert.True(cluster.AsSpan(0x100, Pattern.Length).SequenceEqual(Pattern)); + Assert.Equal(4096 - Pattern.Length, cluster.Count(b => b == 0)); + } + + [Fact] + public void AnIndexBufferOfAnAttributeThatIsNotOpen_HasTheVolumesBufferSize() + { + using var volume = Volume(); + var indexLcn = FirstIndexBuffer(volume); + var index = Read(volume, indexLcn * 4096, 4096, logical: true); + var entry = 0x18 + (int)EndianUtilities.ToUInt32LittleEndian(index, 0x18); + using var result = WithRecordsOn(volume, + Op(NtfsLogOperation.UpdateFileNameAllocation, Enumerable.Repeat((byte)0x44, 0x38).ToArray(), 0x90, 0, [indexLcn], attributeOffset: entry), + Forget()); + + NtfsJournal.Replay(result); + Assert.Equal(0x44, Read(result, indexLcn * 4096, 4096, logical: true)[entry + 0x18]); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalEdgeCaseTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalEdgeCaseTest.cs new file mode 100644 index 000000000..1548794e5 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalEdgeCaseTest.cs @@ -0,0 +1,819 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using DiscUtils; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Edge cases of reading and replaying a journal: damaged or unusual restart pages and checkpoint tables, records +/// at the very end of the log, torn pages, freed pages, and the $LogFile and $MFTMirr locations. +/// +public partial class NtfsJournalEdgeCaseTest +{ + private const ushort Mft = 0x18; + private const ushort Index = 0x40; + private const ushort Data = 0x68; + private const uint Tx = 0x40; + + private static readonly (ushort, uint, ulong, uint, string)[] Attributes = + [ + (Mft, 0x80, 0x0001000000000000UL, 0, ""), + (Index, 0xA0, 0x0005000000000005UL, 4096, "$I30"), + (Data, 0x80, 0x0001000000000006UL, 0, ""), + ]; + + private static readonly Lazy Base = new(() => + { + var stream = new MemoryStream(); + var geometry = Geometry.FromCapacity(16 * 1024 * 1024); + using (var ntfs = NtfsFileSystem.Format(stream, "EDGE", geometry, 0, geometry.TotalSectorsLong)) + { + ntfs.CreateDirectory("big"); + for (var i = 0; i < 150; i++) + { + using var s = ntfs.OpenFile($@"big\file-with-a-long-name-{i:D4}.txt", FileMode.Create); + s.WriteByte((byte)i); + } + } + + return stream.ToArray(); + }); + + private static MemoryStream Volume() => JournalScenario.Copy(Base.Value); + + private static long FreeCluster(Stream volume, int skip = 0) + { + var bytes = ((MemoryStream)volume).ToArray(); + for (var lcn = (bytes.Length / 4096) - 2; lcn > 16; lcn--) + { + if (bytes.AsSpan(lcn * 4096, 4096).ToArray().All(b => b == 0) && skip-- == 0) + { + return lcn; + } + } + + throw new InvalidOperationException("no free cluster"); + } + + private static long FirstIndexBuffer(Stream volume) + { + var bytes = ((MemoryStream)volume).ToArray(); + for (var lcn = 16; lcn < bytes.Length / 4096; lcn++) + { + if (EndianUtilities.ToUInt32LittleEndian(bytes, lcn * 4096) == JournalUtilities.SignatureIndx) + { + return lcn; + } + } + + throw new InvalidOperationException("no index buffer"); + } + + private static byte[] Read(Stream volume, long position, int length, bool logical = false) + { + var bytes = new byte[length]; + JournalUtilities.ReadExactly(volume, position, bytes, 0, length); + if (logical) + { + Assert.True(JournalUtilities.RemoveFixups(bytes, 0, length)); + } + + return bytes; + } + + private static TestLogRecord Raw(long lcn, byte[] redo, byte[]? undo = null, ushort attribute = Data, uint tx = Tx) => new() + { + Redo = NtfsLogOperation.UpdateNonresidentValue, + Undo = NtfsLogOperation.UpdateNonresidentValue, + TransactionId = tx, + RedoData = redo, + UndoData = undo ?? new byte[redo.Length], + TargetAttribute = attribute, + TargetVcn = lcn, + Lcns = [lcn], + }; + + private static TestLogRecord Op(NtfsLogOperation op, byte[] data, ushort attribute, long vcn, long[] lcns, int recordOffset = 0, + int attributeOffset = 0, int blockOffset = 0, NtfsLogOperation undo = NtfsLogOperation.Noop, byte[]? undoData = null) => new() + { + Redo = op, + Undo = undo, + TransactionId = Tx, + RedoData = data, + UndoData = undoData ?? [], + TargetAttribute = attribute, + TargetVcn = vcn, + Lcns = lcns, + RecordOffset = recordOffset, + AttributeOffset = attributeOffset, + ClusterBlockOffset = blockOffset, + }; + + private static TestLogRecord Forget(uint tx = Tx) => + new() { Redo = NtfsLogOperation.ForgetTransaction, Undo = NtfsLogOperation.CompensationLogRecord, TransactionId = tx }; + + private static TestLogRecord DeleteClusters(long lcn) => + new() + { + Redo = NtfsLogOperation.DeleteDirtyClusters, + Undo = NtfsLogOperation.Noop, + TransactionId = Tx, + RedoData = BitConverter.GetBytes(lcn).Concat(BitConverter.GetBytes(1L)).ToArray(), + }; + + private static byte[] RestartRecord(uint major, ulong checkpoint, ulong oat = 0, int oatLength = 0, ulong names = 0, + int namesLength = 0, ulong dirty = 0, int dirtyLength = 0) + { + var r = new byte[0x40]; + EndianUtilities.WriteBytesLittleEndian(major, r, 0x00); + EndianUtilities.WriteBytesLittleEndian(checkpoint, r, 0x08); + EndianUtilities.WriteBytesLittleEndian(oat, r, 0x10); + EndianUtilities.WriteBytesLittleEndian(names, r, 0x18); + EndianUtilities.WriteBytesLittleEndian(dirty, r, 0x20); + EndianUtilities.WriteBytesLittleEndian((uint)oatLength, r, 0x30); + EndianUtilities.WriteBytesLittleEndian((uint)namesLength, r, 0x34); + EndianUtilities.WriteBytesLittleEndian((uint)dirtyLength, r, 0x38); + return r; + } + + private static byte[] Table(int entrySize, int entries, int allocated = -1) + { + var table = new byte[0x18 + (entrySize * entries)]; + EndianUtilities.WriteBytesLittleEndian((ushort)entrySize, table, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)entries, table, 2); + for (var i = 0; i < (allocated < 0 ? entries : allocated); i++) + { + EndianUtilities.WriteBytesLittleEndian(0xFFFFFFFFu, table, 0x18 + (i * entrySize)); + } + + return table; + } + + private static ulong Dump(TestJournalWriter w, NtfsLogOperation op, byte[] table) => + w.Append(new TestLogRecord { Redo = op, Undo = NtfsLogOperation.Noop, TransactionId = 0x18, RedoData = table }); + + /// A journal with a hand-made restart record. + private static MemoryStream WithRestart(Func restart) + { + var volume = Volume(); + var w = new TestJournalWriter(volume); + var checkpoint = w.Append(new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0x18 }); + var lsn = w.AppendRaw(restart(w, checkpoint), LfsRecord.TypeClientRestart, 0, 0); + w.Finish(lsn); + return volume; + } + + private static MemoryStream WithRecords(params TestLogRecord[] records) + { + var volume = Volume(); + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint(Attributes); + foreach (var r in records) + { + w.Append(r); + } + + w.Finish(restart); + return volume; + } + + private static void AssertRefused(Stream volume, string reason) + { + var info = NtfsJournal.Analyze(volume); + Assert.True(info.Status == NtfsJournalStatus.Unsupported, $"expected Unsupported ({reason}), got {info}"); + Assert.True(info.Detail.Contains(reason), $"expected '{reason}' in '{info.Detail}'"); + } + + private static void PatchRestartPages(TestJournalWriter w, Action patch) + { + foreach (var offset in new long[] { 0, TestJournalWriter.PageSize }) + { + var page = new byte[TestJournalWriter.PageSize]; + w.Storage.Read(offset, page, 0, page.Length); + Assert.True(JournalUtilities.RemoveFixups(page, 0, page.Length)); + patch(page); + JournalUtilities.ApplyFixups(page, 0, page.Length); + w.Storage.Write(offset, page, 0, page.Length); + } + } + + [Fact] + public void CheckpointTables_ThatDoNotAddUp_AreRefused() + { + // A table dump shorter than the restart record says. + using (var v = WithRestart((w, c) => RestartRecord(1, c, Dump(w, NtfsLogOperation.OpenAttributeTableDump, Table(0x28, 1)), 0x100))) + { + AssertRefused(v, "shorter than the restart record says"); + } + + // Open attribute entries of the wrong size for the version. + using (var v = WithRestart((w, c) => RestartRecord(1, c, Dump(w, NtfsLogOperation.OpenAttributeTableDump, Table(0x30, 1)), 0x48))) + { + AssertRefused(v, "entries are 48 bytes"); + } + + // A table dump too short to hold a header, and one with an impossible entry size. + using (var v = WithRestart((w, c) => RestartRecord(1, c, Dump(w, NtfsLogOperation.OpenAttributeTableDump, new byte[0x10]), 0x10))) + { + AssertRefused(v, "too short"); + } + + using (var v = WithRestart((w, c) => RestartRecord(1, c, Dump(w, NtfsLogOperation.OpenAttributeTableDump, Table(4, 2)), 0x20))) + { + AssertRefused(v, "implausible header"); + } + + // A dirty page that would cover three clusters. + using (var v = WithRestart((w, c) => RestartRecord(1, c, dirty: Dump(w, NtfsLogOperation.DirtyPageTableDump, Table(0x20 + 24, 1)), dirtyLength: 0x50))) + { + AssertRefused(v, "not a power of two"); + } + + // A dirty page entry that lists more clusters than it holds. + using (var v = WithRestart((w, c) => + { + var table = Table(0x28, 1); + EndianUtilities.WriteBytesLittleEndian(10u, table, 0x18 + 0x0C); + return RestartRecord(1, c, dirty: Dump(w, NtfsLogOperation.DirtyPageTableDump, table), dirtyLength: table.Length); + })) + { + AssertRefused(v, "more clusters than it holds"); + } + + // A negative cluster count, and entries too small to hold any cluster. + using (var v = WithRestart((w, c) => + { + var table = Table(0x28, 1); + EndianUtilities.WriteBytesLittleEndian(0xFFFFFFFFu, table, 0x18 + 0x0C); + return RestartRecord(1, c, dirty: Dump(w, NtfsLogOperation.DirtyPageTableDump, table), dirtyLength: table.Length); + })) + { + AssertRefused(v, "more clusters than it holds"); + } + + using (var v = WithRestart((w, c) => RestartRecord(1, c, dirty: Dump(w, NtfsLogOperation.DirtyPageTableDump, Table(0x20, 1)), dirtyLength: 0x38))) + { + AssertRefused(v, "covers 0 clusters"); + } + + // A dump that is not in the log, and one that is no NTFS record. + using (var v = WithRestart((w, c) => RestartRecord(1, c, w.LsnOf(0x100, 1), 0x40))) + { + AssertRefused(v, "not in the log"); + } + + using (var v = WithRestart((w, c) => RestartRecord(1, c, w.AppendRaw(new byte[8], LfsRecord.TypeClientRecord, 0x18, 0), 0x40))) + { + AssertRefused(v, "too short"); + } + + // A damaged attribute names dump. + using (var v = WithRestart((w, c) => + { + var names = new byte[8]; + EndianUtilities.WriteBytesLittleEndian((ushort)0x18, names, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)0x100, names, 2); + return RestartRecord(1, c, Dump(w, NtfsLogOperation.OpenAttributeTableDump, Table(0x28, 1)), 0x40, + Dump(w, NtfsLogOperation.AttributeNamesDump, names), names.Length); + })) + { + AssertRefused(v, "names dump is damaged"); + } + + // The checkpoint record itself is missing. + using (var v = WithRestart((w, c) => RestartRecord(1, c + 0x2000))) + { + AssertRefused(v, "checkpoint record"); + } + + // An NTFS restart version DiscUtils does not know. + using (var v = WithRestart((w, c) => RestartRecord(2, c))) + { + AssertRefused(v, "version 2.0"); + } + } + + [Fact] + public void ARestartAreaThatPointsAtTheWrongRecord_IsRefused() + { + // At a client record instead of a restart record, and at nothing (LSN 0). + foreach (var (target, reason) in new (Func, string)[] + { + (w => w.Append(new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0x18 }), "not a restart record"), + (_ => 0, "not in the log"), + (w => w.LsnOf(0x100, 1), "not in the log"), + }) + { + using var volume = Volume(); + var w = new TestJournalWriter(volume); + w.Checkpoint(Attributes); + w.Finish(target(w)); + AssertRefused(volume, reason); + } + } + + [Fact] + public void ACheckpointWithAnEmptyDirtyPageTable_StillSetsThePageSize() + { + // No allocated entries, but entries for two clusters per page: records may name two clusters. + using var probe = Volume(); + var a = FreeCluster(probe); + using var volume = WithRestart((w, c) => + { + var oat = Table(0x28, 3); + foreach (var (offset, type) in new[] { (0x18, 0x80u), (0x40, 0xA0u), (0x68, 0x80u) }) + { + TestJournalWriter.OpenAttributeEntryBytes(type, 1, 4096).AsSpan(4).CopyTo(oat.AsSpan(offset + 4)); + } + + var oatLsn = Dump(w, NtfsLogOperation.OpenAttributeTableDump, oat); + var dirty = Dump(w, NtfsLogOperation.DirtyPageTableDump, Table(0x30, 1, allocated: 0)); + var record = Raw(a, Enumerable.Repeat((byte)0x21, 16).ToArray()); + record.Lcns = [a, a - 1]; + w.Append(record); + w.Append(Forget()); + return RestartRecord(1, c, oatLsn, oat.Length, dirty: dirty, dirtyLength: 0x48); + }); + + var info = NtfsJournal.Replay(volume); + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.Equal(0x21, Read(volume, a * 4096, 1)[0]); + } + + [Fact] + public void ResolvedPreparedTransactions_AreKept_AndAnOlderTransactionStartsRedo() + { + using var probe = Volume(); + var x = FreeCluster(probe); + var y = FreeCluster(probe, 1); + var z = FreeCluster(probe, 2); + using var volume = Volume(); + volume.Position = z * 4096; + volume.Write(Enumerable.Repeat((byte)0x33, 8).ToArray(), 0, 8); + var w = new TestJournalWriter(volume); + + // A transaction that started before the checkpoint (its change is on disk) and committed: it starts the + // redo pass, and is not rolled back. + var early = w.Append(Raw(z, Enumerable.Repeat((byte)0x33, 8).ToArray(), tx: 0x90)); + var restart = w.Checkpoint(Attributes, transactions: [(0x90, TransactionState.Committed, early, early)], checkpointLsn: null); + w.Append(Raw(x, Enumerable.Repeat((byte)0x11, 8).ToArray())); + w.Append(new TestLogRecord { Redo = NtfsLogOperation.PrepareTransaction, Undo = NtfsLogOperation.Noop, TransactionId = Tx }); + w.Append(new TestLogRecord { Redo = NtfsLogOperation.CommitTransaction, Undo = NtfsLogOperation.Noop, TransactionId = Tx }); + w.Append(Raw(y, Enumerable.Repeat((byte)0x22, 8).ToArray(), tx: 0x68)); + w.Append(new TestLogRecord { Redo = NtfsLogOperation.CommitTransaction, Undo = NtfsLogOperation.Noop, TransactionId = 0x68 }); + w.Finish(restart); + + var info = NtfsJournal.Replay(volume); + Assert.Equal(0, info.TransactionsToUndo); + Assert.Equal(0x11, Read(volume, x * 4096, 1)[0]); + Assert.Equal(0x22, Read(volume, y * 4096, 1)[0]); + Assert.Equal(0x33, Read(volume, z * 4096, 1)[0]); + } + + [Fact] + public void AnAttributeOpenedWithTooLittleData_IsRefused() + { + using var volume = WithRecords(new TestLogRecord + { + Redo = NtfsLogOperation.OpenNonresidentAttribute, Undo = NtfsLogOperation.Noop, TransactionId = Tx, RedoData = new byte[8], TargetAttribute = 0x90, + }); + AssertRefused(volume, "too little data"); + } + + [Theory] + [InlineData("page size")] + [InlineData("restart area")] + [InlineData("no clients")] + [InlineData("client in use")] + public void RestartPages_ThatAreNotPlausible_AreNotUsed(string damage) + { + using var volume = WithRecords(Forget()); + var w = new TestJournalWriter(volume); + PatchRestartPages(w, page => + { + switch (damage) + { + case "page size": EndianUtilities.WriteBytesLittleEndian(100u, page, 0x14); break; + case "restart area": EndianUtilities.WriteBytesLittleEndian((ushort)0x2C, page, 0x18); break; + case "no clients": EndianUtilities.WriteBytesLittleEndian((ushort)0, page, 0x30 + 0x08); break; + case "client in use": EndianUtilities.WriteBytesLittleEndian((ushort)3, page, 0x30 + 0x0C); break; + } + }); + AssertRefused(volume, "no valid restart page"); + } + + [Fact] + public void InvalidRecordLengthsAreRefused_AndATornContinuationPageEndsTheLog() + { + foreach (var damage in new[] { "length", "negative length", "oversized NTFS payload", "continuation" }) + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"x\y"), setup: ntfs => + { + for (var i = 0; i < 60; i++) + { + ntfs.CreateDirectory($"pre{i}"); + } + }); + using var volume = JournalScenario.Copy(scenario.After); + var writer = scenario.WriteJournal(volume, committed: true); + + // Find the commit (the last record) or the first record that spans pages, and damage the log there. + var lsn = writer.LastLsn; + var offset = writer.LsnToOffsetForTest(lsn); + var page = new byte[TestJournalWriter.PageSize]; + if (damage != "continuation") + { + var pageOffset = offset & ~(long)(TestJournalWriter.PageSize - 1); + writer.Storage.Read(pageOffset, page, 0, page.Length); + Assert.True(JournalUtilities.RemoveFixups(page, 0, page.Length)); + var length = damage switch + { + "length" => 0x7FFFFFFFu, + "negative length" => 0xFFFFFFFFu, + _ => 0x100000u, + }; + EndianUtilities.WriteBytesLittleEndian(length, page, (int)(offset - pageOffset) + 0x18); + JournalUtilities.ApplyFixups(page, 0, page.Length); + writer.Storage.Write(pageOffset, page, 0, page.Length); + } + else + { + // The page a big record (an index buffer image) continues on, after some records of the transaction. + var log = LfsLog.Open(writer.Storage, out _); + var big = log.ReadForward(writer.LsnOf(writer.FirstPageOffset + 0x40, 1)) + .Where(r => r.TransactionId == JournalScenario.Transaction) + .Skip(1) + .First(r => r.ClientData.Length > TestJournalWriter.PageSize); + var continuation = (writer.LsnToOffsetForTest(big.Lsn) & ~(long)(TestJournalWriter.PageSize - 1)) + TestJournalWriter.PageSize; + writer.Storage.Read(continuation, page, 0, page.Length); + page[0x3FE] ^= 0xFF; + writer.Storage.Write(continuation, page, 0, page.Length); + } + + if (damage == "continuation") + { + // The transaction lost its end: it is rolled back. + var info = NtfsJournal.Analyze(volume); + Assert.True(info.TransactionsToUndo == 1, $"{damage}: {info}"); + } + else + { + AssertReplayRefused(volume, "invalid client data length"); + } + } + } + + [Fact] + public void ARecordHeaderThatDoesNotFitAtTheEndOfTheLog_ContinuesOnTheFirstPage() + { + using var volume = Volume(); + var target = FreeCluster(volume); + var size = new TestJournalWriter(volume).FileSize; + var w = new TestJournalWriter(volume, startOffset: size - (2 * TestJournalWriter.PageSize)); + var restart = w.Checkpoint(Attributes); + + // A filler that ends 16 bytes before the end of the last page: the next header cannot fit there. + var fillerEnd = size - 0x10; + var pagesCrossed = (fillerEnd / TestJournalWriter.PageSize) - (w.Position / TestJournalWriter.PageSize); + var filler = (int)(fillerEnd - w.Position - 0x30 - 0x20 - (0x40 * pagesCrossed)); + w.Append(new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0x18, RedoData = new byte[filler & ~7] }); + Assert.Equal(fillerEnd, w.Position); + var change = w.Append(Raw(target, Enumerable.Repeat((byte)0x5A, 32).ToArray())); + Assert.Equal(w.FirstPageOffset + 0x40, w.LsnToOffsetForTest(change)); + w.Append(Forget()); + w.Finish(restart); + + NtfsJournal.Replay(volume); + Assert.Equal(0x5A, Read(volume, target * 4096, 1)[0]); + } + + [Fact] + public void FreedPages_AreNotTouched_ByRedoOrUndo() + { + using var volume = Volume(); + var indexLcn = FirstIndexBuffer(volume); + var index = Read(volume, indexLcn * 4096, 4096, logical: true); + var entry = 0x18 + (int)EndianUtilities.ToUInt32LittleEndian(index, 0x18); + var rawLcn = FreeCluster(volume); + var bitsLcn = FreeCluster(volume, 1); + var original = ((MemoryStream)volume).ToArray(); + var mftLcn = 0L; + using (var ntfs = new NtfsFileSystem(JournalScenario.Copy(original), NtfsJournalMode.Ignore)) + { + mftLcn = ntfs.PathToClusters("$MFT").First().Offset + 4; + } + + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint(Attributes); + w.Append(Op(NtfsLogOperation.UpdateResidentValue, new byte[8], Mft, 4, [mftLcn], 0x38, 0x18, undo: NtfsLogOperation.UpdateResidentValue, undoData: new byte[8])); + w.Append(Op(NtfsLogOperation.UpdateFileNameAllocation, new byte[0x38], Index, 0, [indexLcn], attributeOffset: entry)); + w.Append(Op(NtfsLogOperation.UpdateNonresidentValue, new byte[4096], Index, 9999, [indexLcn + 1000])); + w.Append(Op(NtfsLogOperation.SetBitsInNonresidentBitMap, BitConverter.GetBytes(1L << 32), Data, 77, [bitsLcn])); + w.Append(Raw(rawLcn, Enumerable.Repeat((byte)0x77, 16).ToArray())); + foreach (var lcn in new[] { mftLcn, indexLcn, indexLcn + 1000, bitsLcn, rawLcn }) + { + w.Append(DeleteClusters(lcn)); + } + + w.Finish(restart); + var info = NtfsJournal.Replay(volume); + Assert.Equal(NtfsJournalStatus.NothingToReplay, info.Status); + Assert.True(original.AsSpan(0, (int)(mftLcn * 4096)).SequenceEqual(((MemoryStream)volume).ToArray().AsSpan(0, (int)(mftLcn * 4096)))); + } + + [Fact] + public void PagesThatAreUpToDate_EmptyWrites_AndBitsAlreadySet_ChangeNothing() + { + using var volume = Volume(); + var indexLcn = FirstIndexBuffer(volume); + var bitsLcn = FreeCluster(volume); + volume.Position = bitsLcn * 4096; + volume.WriteByte(0xFF); + + // The index buffer already carries an LSN newer than the record. + var index = Read(volume, indexLcn * 4096, 4096, logical: true); + EndianUtilities.WriteBytesLittleEndian(ulong.MaxValue - 1, index, 8); + var protectedIndex = (byte[])index.Clone(); + JournalUtilities.ApplyFixups(protectedIndex, 0, protectedIndex.Length); + volume.Position = indexLcn * 4096; + volume.Write(protectedIndex, 0, protectedIndex.Length); + var entry = 0x18 + (int)EndianUtilities.ToUInt32LittleEndian(index, 0x18); + + using var result = WithRecordsOn(volume, + Op(NtfsLogOperation.UpdateFileNameAllocation, new byte[0x38], Index, 0, [indexLcn], attributeOffset: entry), + Op(NtfsLogOperation.UpdateNonresidentValue, [], Data, 5, [bitsLcn]), + Op(NtfsLogOperation.SetBitsInNonresidentBitMap, BitConverter.GetBytes(0x0000000400000000L), Data, 6, [bitsLcn]), + Forget()); + Assert.Equal(NtfsJournalStatus.NothingToReplay, NtfsJournal.Analyze(result).Status); + } + + private static MemoryStream WithRecordsOn(MemoryStream volume, params TestLogRecord[] records) + { + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint(Attributes); + foreach (var r in records) + { + w.Append(r); + } + + w.Finish(restart); + return volume; + } + + [Fact] + public void AnIndexAllocationWithoutABufferSize_UsesTheVolumes() + { + using var volume = Volume(); + var indexLcn = FirstIndexBuffer(volume); + var index = Read(volume, indexLcn * 4096, 4096, logical: true); + var entry = 0x18 + (int)EndianUtilities.ToUInt32LittleEndian(index, 0x18); + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint([(Index, 0xA0, 0x0005000000000005UL, 0, "$I30")]); + w.Append(Op(NtfsLogOperation.UpdateFileNameAllocation, Enumerable.Repeat((byte)0x44, 0x38).ToArray(), Index, 0, [indexLcn], attributeOffset: entry)); + w.Append(Forget()); + w.Finish(restart); + + NtfsJournal.Replay(volume); + Assert.Equal(0x44, Read(volume, indexLcn * 4096, 4096, logical: true)[entry + 0x18]); + } + + [Fact] + public void PagesBeyondTheirClusters_OrPartlyFreed_AreRefused() + { + using var probe = Volume(); + long mftLcn; + using (var ntfs = new NtfsFileSystem(JournalScenario.Copy(Base.Value), NtfsJournalMode.Ignore)) + { + mftLcn = ntfs.PathToClusters("$MFT").First().Offset; + } + + // A file record that would run past the cluster the record names. + using (var volume = WithRecords(Op(NtfsLogOperation.UpdateResidentValue, new byte[8], Mft, 0, [mftLcn], 0x38, 0x18, blockOffset: 7))) + { + AssertRefused(volume, "beyond the clusters it names"); + } + + // An 8K index buffer whose second cluster was freed. + var a = FreeCluster(probe); + var b = a - 1; + using (var volume = Volume()) + { + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint( + [(Index, 0xA0, 0x0005000000000005UL, 8192, "$I30")], + [(Index, 0, 0, [a, b])], + clustersPerPage: 2); + w.Append(Op(NtfsLogOperation.UpdateFileNameAllocation, new byte[0x38], Index, 0, [a, b], attributeOffset: 0x40)); + w.Append(DeleteClusters(b)); + w.Append(Forget()); + w.Finish(restart); + AssertRefused(volume, "partly freed"); + } + } + + [Fact] + public void FileRecordsZeroToThree_AreMirrored_AlsoWhenTheySpanSmallClusters() + { + // 512-byte clusters: the first four records span eight clusters, most of which the replay does not change. + using var volume = NtfsJournalWindowsSampleTest.LoadVolume("s512"); + NtfsJournal.Replay(volume); + var boot = Read(volume, 0, 512); + var mft = EndianUtilities.ToInt64LittleEndian(boot, 0x30); + var mirror = EndianUtilities.ToInt64LittleEndian(boot, 0x38); + var record1 = Read(volume, (mft * 512) + 1024, 1024, logical: true); + var changed = (byte[])record1.Clone(); + var used = (int)EndianUtilities.ToUInt32LittleEndian(changed, 0x18); + + // Continue with LSNs past the ones Windows left on the pages. + var w = new TestJournalWriter(volume, sequence: 1000); + var restart = w.Checkpoint([(Mft, 0x80, 0x0001000000000000UL, 0, "")]); + w.Append(Op(NtfsLogOperation.UpdateResidentValue, BitConverter.GetBytes(0x01D1111111111111L), Mft, 2, [mft + 2, mft + 3], + EndianUtilities.ToUInt16LittleEndian(changed, 0x14), 0x18, undo: NtfsLogOperation.UpdateResidentValue, undoData: changed.AsSpan(0x50, 8).ToArray())); + w.Append(Forget()); + w.Finish(restart); + + NtfsJournal.Replay(volume); + Assert.True(used > 0); + Assert.True(Read(volume, mft * 512, 4096).AsSpan().SequenceEqual(Read(volume, mirror * 512, 4096))); + Assert.Equal(0x01D1111111111111L, EndianUtilities.ToInt64LittleEndian(Read(volume, (mft * 512) + 1024, 1024, logical: true), 0x50)); + } + + [Fact] + public void LogFileRuns_AreRead_AndAccessOutsideThemIsRefused() + { + // Two runs, the second before the first (a negative offset), then the end. + var runs = new byte[] { 0x21, 0x10, 0x00, 0x01, 0x11, 0x08, 0xF0, 0x00 }; + var extents = LogFileStorage.ParseRuns(runs, 0, runs.Length, 1024, "$LogFile"); + Assert.Equal([(0L, 0x100L, 0x10L), (0x10L, 0xF0L, 8L)], extents); + + // An eight-byte offset (nothing to sign-extend). + byte[] wide = [0x81, 0x02, 0x20, 0, 0, 0, 0, 0, 0, 0, 0]; + Assert.Equal([(0L, 0x20L, 2L)], LogFileStorage.ParseRuns(wide, 0, wide.Length, 1024, "$LogFile")); + + using var volume = Volume(); + var layout = JournalVolumeLayout.Read(volume); + var storage = LogFileStorage.Open(volume, layout); + Assert.Throws(() => storage.Read(storage.Length, new byte[16], 0, 16)); + Assert.Throws(() => storage.Read(-1, new byte[16], 0, 16)); + Assert.Throws(() => storage.Read(long.MaxValue, new byte[16], 0, 16)); + + // $LogFile's data longer than its clusters: the end cannot be read. + long position; + using (var ntfs = new NtfsFileSystem(JournalScenario.Copy(Base.Value), NtfsJournalMode.Ignore)) + { + position = (ntfs.PathToClusters("$MFT").First().Offset * 4096) + 2048; + } + + var record = Read(volume, position, 1024, logical: true); + var attr = (int)EndianUtilities.ToUInt16LittleEndian(record, 0x14); + while (EndianUtilities.ToUInt32LittleEndian(record, attr) != 0x80) + { + attr += (int)EndianUtilities.ToUInt32LittleEndian(record, attr + 4); + } + + EndianUtilities.WriteBytesLittleEndian(EndianUtilities.ToUInt64LittleEndian(record, attr + 0x30) * 2, record, attr + 0x30); + JournalUtilities.ApplyFixups(record, 0, record.Length); + volume.Position = position; + volume.Write(record, 0, record.Length); + AssertReplayRefused(volume, "invalid allocated, data or initialized lengths"); + } + + [Theory] + [InlineData(0x7FFFFFFFL)] + [InlineData(-1L)] + public void AMirrorClusterOutsideTheVolume_IsNoFallback(long mirrorCluster) + { + using var volume = Volume(); + var boot = Read(volume, 0, 512); + EndianUtilities.WriteBytesLittleEndian(mirrorCluster, boot, 0x38); + volume.Position = 0; + volume.Write(boot, 0, boot.Length); + long position; + using (var ntfs = new NtfsFileSystem(JournalScenario.Copy(Base.Value), NtfsJournalMode.Ignore)) + { + position = (ntfs.PathToClusters("$MFT").First().Offset * 4096) + 2048; + } + + volume.Position = position + 0x1FE; + volume.WriteByte(0x55); + AssertRefused(volume, "Neither"); + } + + [Fact] + public void SmallThings_NotAVolume_FixupLayout_AndARecordOffsetPastTheEnd() + { + Assert.Throws(() => NtfsJournal.Analyze(new MemoryStream(new byte[64 * 1024]))); + + var record = new byte[1000]; + EndianUtilities.WriteBytesLittleEndian((ushort)0x30, record, 4); + EndianUtilities.WriteBytesLittleEndian((ushort)3, record, 6); + Assert.False(JournalUtilities.HasFixupLayout(record, 0, 1000)); + Assert.False(JournalUtilities.RemoveFixups(record, 0, 1000)); + + // An attribute offset past the end marker of a file record. + var page = new byte[1024]; + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureFile, page, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)0x38, page, 0x14); + EndianUtilities.WriteBytesLittleEndian(0xFFFFFFFFu, page, 0x38); + EndianUtilities.WriteBytesLittleEndian(0x40u, page, 0x18); + var log = new TestLogRecord { Redo = NtfsLogOperation.DeleteAttribute, RecordOffset = 0x60, Lcns = [1] }; + var parsed = NtfsLogRecord.TryParse(new LfsRecord { Lsn = 1, ClientData = log.ClientData() }, out _); + Assert.Throws(() => NtfsLogActions.ApplyToFileRecord(parsed.RedoOperation, parsed, page, parsed.RedoData)); + + // Data stored only partly: the rest counts as zeros. + var partial = new TestLogRecord { Redo = NtfsLogOperation.UpdateResidentValue, RedoData = [1, 2, 3, 4], RedoLength = 12, Lcns = [1] }; + var p2 = NtfsLogRecord.TryParse(new LfsRecord { Lsn = 1, ClientData = partial.ClientData() }, out _); + Assert.Equal(new byte[] { 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0 }, p2.RedoData.ToArray()); + } + + [Fact] + public void AClusterUsedAsTwoKindsOfPage_IsRefused() + { + using var probe = Volume(); + var x = FreeCluster(probe); + + // The same bytes as a file record and as an index buffer. + using (var volume = WithRecords( + Op(NtfsLogOperation.InitializeFileRecordSegment, new byte[0x30], Mft, 0, [x]), + Op(NtfsLogOperation.UpdateFileNameAllocation, new byte[0x38], Index, 0, [x], attributeOffset: 0x40))) + { + AssertRefused(volume, "both as FileRecord and as IndexBuffer"); + } + + // A cluster written raw and as a file record. + using (var volume = WithRecords( + Raw(x, new byte[8]), + Op(NtfsLogOperation.InitializeFileRecordSegment, new byte[0x30], Mft, 0, [x]))) + { + AssertRefused(volume, "both as Raw and as FileRecord"); + } + } + + [Fact] + public void ATornFileRecord_IsOnlyRebuilt_NeverChanged() + { + using var volume = Volume(); + var x = FreeCluster(volume); + var torn = new byte[1024]; + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureFile, torn, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)0x30, torn, 4); + EndianUtilities.WriteBytesLittleEndian((ushort)3, torn, 6); + EndianUtilities.WriteBytesLittleEndian((ushort)5, torn, 0x30); + EndianUtilities.WriteBytesLittleEndian((ushort)5, torn, 0x1FE); + EndianUtilities.WriteBytesLittleEndian((ushort)4, torn, 0x3FE); + volume.Position = x * 4096; + volume.Write(torn, 0, torn.Length); + using var result = WithRecordsOn(volume, Op(NtfsLogOperation.UpdateResidentValue, new byte[8], Mft, 0, [x], 0x38, 0x18), Forget()); + AssertRefused(result, "damaged"); + } + + [Fact] + public void ANamesDumpWithoutItsTerminator_IsRead() + { + using var volume = WithRestart((w, c) => + { + var names = new byte[4 + 8]; + EndianUtilities.WriteBytesLittleEndian((ushort)0x18, names, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)8, names, 2); + Encoding.Unicode.GetBytes("$I30").CopyTo(names, 4); + return RestartRecord(1, c, Dump(w, NtfsLogOperation.OpenAttributeTableDump, Table(0x28, 1)), 0x40, + Dump(w, NtfsLogOperation.AttributeNamesDump, names), names.Length); + }); + Assert.Equal(NtfsJournalStatus.NothingToReplay, NtfsJournal.Analyze(volume).Status); + } + + [Fact] + public void AnUndoOnAFreedCluster_IsSkipped() + { + using var volume = Volume(); + var x = FreeCluster(volume); + var after = Enumerable.Repeat((byte)0x66, 16).ToArray(); + volume.Position = x * 4096; + volume.Write(after, 0, after.Length); + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint(Attributes); + var change = w.Append(Raw(x, after)); + var free = DeleteClusters(x); + free.UndoNextLsn = change; + w.Append(free); + w.Finish(restart); + + var info = NtfsJournal.Replay(volume); + Assert.Equal(NtfsJournalStatus.NothingToReplay, info.Status); + Assert.True(Read(volume, x * 4096, 16).SequenceEqual(after)); + } + + [Fact] + public void AFixupArrayThatDoesNotDivideTheRecord_IsNoLayout() + { + var record = new byte[1000]; + EndianUtilities.WriteBytesLittleEndian((ushort)0x30, record, 4); + EndianUtilities.WriteBytesLittleEndian((ushort)4, record, 6); + Assert.False(JournalUtilities.HasFixupLayout(record, 0, 1000)); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalHibernationTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalHibernationTest.cs new file mode 100644 index 000000000..d83102995 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalHibernationTest.cs @@ -0,0 +1,359 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using DiscUtils; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Setup; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +public partial class NtfsJournalEdgeCaseTest +{ + private const string HibernationFile = @"\HIBERFIL.SYS"; + private const string ActiveHibernation = "active Windows hibernation signature"; + private const string IndeterminateHibernation = "hibernation state is indeterminate"; + + [Theory] + [InlineData("hibr")] + [InlineData("HIBR")] + [InlineData("HORM")] + public void ActiveHibernationBlocksEveryPersistentReplayEntryPoint(string signature) + { + using var dirty = Volume(); + PutHibernationFile(dirty, HibernationHeader(signature)); + WriteRecoverableJournal(dirty); + using var volume = new ReplayTrackingStream(dirty.ToArray(), failFlush: 0); + var before = volume.ToArray(); + + AssertReplayRefused(volume, ActiveHibernation); + Assert.Contains(ActiveHibernation, Assert.Throws(() => new NtfsFileSystem(volume, NtfsJournalMode.Replay)).Message); + + SetupHelper.RegisterAssembly(typeof(NtfsFileSystem).Assembly); + var factory = FileSystemManager.DetectFileSystems(volume).Single(f => f.Name == "NTFS"); + var parameters = new FileSystemParameters { Parameters = { { "NtfsJournalMode", "Replay" } } }; + Assert.Contains(ActiveHibernation, Assert.Throws(() => factory.Open(volume, parameters)).Message); + + Assert.Throws(() => new NtfsFileSystem(volume, NtfsJournalMode.Refuse)); + Assert.Throws(() => new NtfsFileSystem(volume)); + Assert.Empty(volume.Events); + Assert.True(before.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void ActiveHibernationAlsoBlocksAResetWithoutMetadataChanges() + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + var writer = new TestJournalWriter(volume); + writer.Finish(writer.Checkpoint([])); + + AssertReplayRefused(volume, ActiveHibernation); + } + + [Fact] + public void HibernationSuppressesTheOverlayAndAllowsRawReadOnlyInspection() + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + var target = WriteRecoverableJournal(volume); + var before = volume.ToArray(); + using var readOnly = new MemoryStream(before, writable: false); + + var info = NtfsJournal.ReplayInMemory(readOnly, out var overlay); + Assert.Equal(NtfsJournalStatus.Unsupported, info.Status); + Assert.Contains(ActiveHibernation, info.Detail); + Assert.Null(overlay); + + using var fs = new NtfsFileSystem(readOnly); + Assert.False(fs.CanWrite); + Assert.Equal(NtfsJournalStatus.Unsupported, fs.JournalInfo.Status); + Assert.False(fs.JournalInfo.ReplayedInMemory); + Assert.True(fs.FileExists(HibernationFile)); + Assert.Equal(0, Read(fs.RawStream, target * 4096, 1)[0]); + Assert.Throws(() => new NtfsFileSystem(readOnly, NtfsJournalMode.Refuse)); + + using var ignored = new NtfsFileSystem(readOnly, NtfsJournalMode.Ignore); + Assert.Null(ignored.JournalInfo); + Assert.False(ignored.CanWrite); + Assert.True(before.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("wake")] + [InlineData("WAKE")] + public void MissingOrInactiveLocalHeaderAllowsOrdinaryRecovery(string? signature) + { + using var volume = Volume(); + if (signature != null) + { + var bytes = new byte[8192]; + Encoding.ASCII.GetBytes(signature).CopyTo(bytes, 0); + bytes[4096] = 0xA5; + PutHibernationFile(volume, bytes); + } + else + { + // A copy outside the root supplies no local indication. + using var fs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore); + using var file = fs.OpenFile(@"big\hiberfil.sys", FileMode.Create, FileAccess.Write); + file.Write(HibernationHeader("HIBR"), 0, 4096); + } + + var target = WriteRecoverableJournal(volume); + var before = volume.ToArray(); + var info = NtfsJournal.Analyze(volume); + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.True(info.CanReplayPersistently); + Assert.True(before.AsSpan().SequenceEqual(volume.ToArray())); + Assert.True(NtfsJournal.Replay(volume).Replayed); + Assert.Equal(Pattern, Read(volume, target * 4096, Pattern.Length)); + } + + [Theory] + [InlineData(0)] + [InlineData(4)] + [InlineData(4095)] + public void AShortHibernationFileIsIndeterminate(int size) + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR").AsSpan(0, size).ToArray()); + WriteRecoverableJournal(volume); + + AssertReplayRefused(volume, IndeterminateHibernation); + } + + [Theory] + [InlineData("rstr", "resume-in-progress signature")] + [InlineData("RSTR", "resume-in-progress signature")] + [InlineData("other", "unrecognized nonzero header")] + [InlineData("", "unrecognized nonzero header")] + public void AnUnrecognizedOrInterruptedResumeHeaderIsIndeterminate(string signature, string reason) + { + using var volume = Volume(); + var header = HibernationHeader(signature); + header[^1] = 0xA5; + PutHibernationFile(volume, header); + WriteRecoverableJournal(volume); + + AssertReplayRefused(volume, reason); + } + + [Theory] + [InlineData(false, "initialized header")] + [InlineData(true, "invalid header cluster mapping")] + public void AnUntrustworthyHibernationDataMappingIsIndeterminate(bool invalidRun, string reason) + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + var layout = JournalVolumeLayout.Read(volume); + using (var fs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore)) + { + var entry = fs.GetDirectoryEntry(HibernationFile)!.Value; + var mft = fs.GetFile(entry.Reference).Context.Mft; + var record = mft.GetRecord(entry.Reference); + var data = (NonResidentAttributeRecord)record.GetAttribute(AttributeType.Data); + if (invalidRun) + { + data.DataRuns[0].RunOffset = layout.TotalClusters; + } + else + { + data.InitializedDataLength = 0; + } + + mft.WriteRecord(record); + } + + WriteRecoverableJournal(volume); + AssertReplayRefused(volume, reason); + } + + [Theory] + [InlineData(false, false, "injected read failure")] + [InlineData(false, true, "Unexpected end of the volume")] + [InlineData(true, false, "injected read failure")] + public void UnreadableHibernationMetadataCannotBeTreatedAsMissing(bool rootRecord, bool shortRead, string reason) + { + using var volume = Volume(); + var headerLcn = PutHibernationFile(volume, new byte[4096]); + WriteRecoverableJournal(volume); + var layout = JournalVolumeLayout.Read(volume); + var fault = rootRecord ? layout.MftCluster * layout.BytesPerCluster + MasterFileTable.RootDirIndex * layout.MftRecordSize : + headerLcn * layout.BytesPerCluster; + using var unreadable = new HibernationReadFault(volume.ToArray(), fault, shortRead); + + AssertReplayRefused(unreadable, reason); + Assert.Contains(IndeterminateHibernation, NtfsJournal.Analyze(unreadable).Detail); + } + + [Fact] + public void AnInvalidRootIndexCannotBeTreatedAsAnEmptyDirectory() + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + using (var fs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore)) + { + var mft = fs.GetDirectory(MasterFileTable.RootDirIndex).Context.Mft; + var record = mft.GetRecord(MasterFileTable.RootDirIndex, ignoreMagic: false); + var index = (ResidentAttributeRecord)record.GetAttribute(AttributeType.IndexRoot, "$I30"); + var endOfEntries = new byte[4]; + index.DataBuffer.Read(IndexRoot.HeaderOffset + 4, endOfEntries, 0, endOfEntries.Length); + index.DataBuffer.Write(IndexRoot.HeaderOffset, endOfEntries, 0, endOfEntries.Length); + mft.WriteRecord(record); + } + + WriteRecoverableJournal(volume); + AssertReplayRefused(volume, "directory index node has no end entry"); + } + + [Fact] + public void AHibernationIndexEntryWithAnInconsistentFileRecordIsRefused() + { + using var volume = Volume(); + PutHibernationFile(volume, new byte[4096]); + using (var fs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore)) + { + var entry = fs.GetDirectoryEntry(HibernationFile)!.Value; + var mft = fs.GetFile(entry.Reference).Context.Mft; + var record = mft.GetRecord(entry.Reference); + var name = (ResidentAttributeRecord)record.GetAttribute(AttributeType.FileName); + name.DataBuffer.Write(0x42, [(byte)'X'], 0, 1); + mft.WriteRecord(record); + } + + WriteRecoverableJournal(volume); + AssertReplayRefused(volume, "inconsistent file record"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ASignatureInEitherTheRawOrTheStagedViewBlocksReplay(bool activeBeforeRedo) + { + using var volume = Volume(); + var lcn = PutHibernationFile(volume, HibernationHeader(activeBeforeRedo ? "HIBR" : "")); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(lcn, activeBeforeRedo ? new byte[4] : Encoding.ASCII.GetBytes("HIBR"))); + writer.Append(Forget()); + writer.Finish(restart); + + using var tracked = new ReplayTrackingStream(volume.ToArray(), failFlush: 0); + var reason = activeBeforeRedo ? ActiveHibernation : $"{ActiveHibernation} in the staged replay view"; + AssertReplayRefused(tracked, reason); + Assert.Equal(NtfsJournalStatus.Unsupported, NtfsJournal.ReplayInMemory(tracked, out var overlay).Status); + Assert.Null(overlay); + Assert.Empty(tracked.Events); + } + + [Fact] + public void ARepairableTornMftBootstrapCannotHideTheHibernationFile() + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + var writer = new TestJournalWriter(volume); + var mft = writer.Layout.MftCluster; + var offset = mft * writer.Layout.BytesPerCluster; + var record = Read(volume, offset, writer.Layout.MftRecordSize, logical: true); + var restart = writer.Checkpoint(Attributes); + writer.Append(Op(NtfsLogOperation.InitializeFileRecordSegment, record, Mft, 0, [mft])); + writer.Append(Forget()); + writer.Finish(restart); + + volume.Position = offset + record.Length - 2; + var trailer = volume.ReadByte(); + volume.Position--; + volume.WriteByte((byte)(trailer ^ 0xFF)); + + AssertReplayRefused(volume, ActiveHibernation); + } + + [Fact] + public void APartitionStreamWithActiveHibernationIsAlsoRefused() + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + WriteRecoverableJournal(volume); + using var disk = new MemoryStream(); + disk.Position = 512; + volume.WriteTo(disk); + using var partition = new SubStream(disk, 512, volume.Length); + var before = disk.ToArray(); + + Assert.Contains(ActiveHibernation, Assert.Throws(() => NtfsJournal.Replay(partition)).Message); + Assert.True(before.AsSpan().SequenceEqual(disk.ToArray())); + } + + [Fact] + public void AReplayCallWithACleanJournalMakesNoChanges() + { + using var volume = Volume(); + PutHibernationFile(volume, HibernationHeader("HIBR")); + var before = volume.ToArray(); + + var info = NtfsJournal.Replay(volume); + Assert.Equal(NtfsJournalStatus.Clean, info.Status); + Assert.False(info.Replayed); + Assert.True(before.AsSpan().SequenceEqual(volume.ToArray())); + } + + private static byte[] HibernationHeader(string signature) + { + var bytes = new byte[4096]; + Encoding.ASCII.GetBytes(signature).CopyTo(bytes, 0); + return bytes; + } + + private static long PutHibernationFile(Stream volume, byte[] bytes) + { + using var fs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore); + using (var file = fs.OpenFile(HibernationFile, FileMode.Create, FileAccess.Write)) + { + file.Write(bytes, 0, bytes.Length); + } + + fs.SetAttributes(HibernationFile, FileAttributes.Hidden | FileAttributes.System); + return fs.PathToClusters(HibernationFile).FirstOrDefault().Offset; + } + + private static long WriteRecoverableJournal(MemoryStream volume) + { + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(target, Pattern)); + writer.Append(Forget()); + writer.Finish(restart); + return target; + } + + private sealed class HibernationReadFault(byte[] image, long faultOffset, bool shortRead) : MemoryStream(image) + { + public override int Read(byte[] buffer, int offset, int count) => + FailRead(count) ? 0 : base.Read(buffer, offset, count); + +#if NET + public override int Read(Span buffer) => FailRead(buffer.Length) ? 0 : base.Read(buffer); +#endif + + private bool FailRead(int count) + { + if (Position > faultOffset || count <= faultOffset - Position) + { + return false; + } + + if (!shortRead) + { + throw new IOException("injected read failure"); + } + + return true; + } + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalInterruptionTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalInterruptionTest.cs new file mode 100644 index 000000000..fcfb86c4e --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalInterruptionTest.cs @@ -0,0 +1,403 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace LibraryTests.Ntfs.Journal; + +public partial class NtfsJournalEdgeCaseTest +{ + // The fixture issues these events in order: + // 1 mirror write, 2 mirror barrier, 3 primary MFT write, 4 raw metadata write, 5 metadata barrier, + // 6 alternate restart write, 7 alternate barrier, 8 source restart write, 9 final barrier. + [Theory] + [InlineData(1, 0, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(1, 512, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(1, 4096, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(2, 0, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(2, 0, true, NtfsJournalStatus.NeedsReplay)] + [InlineData(3, 0, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(3, 512, false, NtfsJournalStatus.Unsupported)] + [InlineData(3, 4096, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(4, 0, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(4, 512, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(4, 4096, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(5, 0, false, NtfsJournalStatus.NeedsReplay)] + [InlineData(5, 0, true, NtfsJournalStatus.NothingToReplay)] + [InlineData(6, 0, false, NtfsJournalStatus.NothingToReplay)] + [InlineData(6, 512, false, NtfsJournalStatus.NothingToReplay)] + [InlineData(6, 4096, false, NtfsJournalStatus.NothingToReplay)] + [InlineData(7, 0, false, NtfsJournalStatus.NothingToReplay)] + [InlineData(7, 0, true, NtfsJournalStatus.NothingToReplay)] + [InlineData(8, 0, false, NtfsJournalStatus.NothingToReplay)] + [InlineData(8, 512, false, NtfsJournalStatus.Clean)] + [InlineData(8, 4096, false, NtfsJournalStatus.Clean)] + [InlineData(9, 0, false, NtfsJournalStatus.NothingToReplay)] + [InlineData(9, 0, true, NtfsJournalStatus.Clean)] + [InlineData(0, 0, false, NtfsJournalStatus.Clean)] // No fault. Reopen after the final successful barrier. + public void RedoRetryUsesThePersistedImageAtEachWriteAndBarrier(int cut, int persistedPrefix, bool completedBarrier, + NtfsJournalStatus expected) + { + using var dirty = Volume(); + var (writer, rawLcn, updateLsn, attribute) = WriteRedoInterruptionFixture(dirty); + var mft = writer.Layout.MftCluster * 4096; + var mirror = writer.Layout.MftMirrorCluster * 4096; + using var fs = new NtfsFileSystem(dirty, NtfsJournalMode.Ignore); + var logStart = fs.PathToClusters("$LogFile").First().Offset * 4096; + var writes = new[] + { + (Step: 1, Barrier: 2, Position: mirror), + (Step: 3, Barrier: 5, Position: mft), + (Step: 4, Barrier: 5, Position: rawLcn * 4096), + (Step: 6, Barrier: 7, Position: logStart + 4096), + (Step: 8, Barrier: 9, Position: logStart), + }; + + var initial = dirty.ToArray(); + using var device = new CrashImageStream(initial, cut, persistedPrefix, completedBarrier); + if (cut == 0) + { + Assert.True(NtfsJournal.Replay(device).Replayed); + } + else + { + Assert.Same(device.Failure, Assert.Throws(() => NtfsJournal.Replay(device))); + } + + // Each write is for a different cluster. Check the exact stable image, including untouched bytes. + var events = new (long Position, int Count)[9]; // Default entry denotes a flush. + using var completed = JournalScenario.Copy(initial); + Assert.True(NtfsJournal.Replay(completed).Replayed); + var expectedImage = (byte[])initial.Clone(); + foreach (var (step, barrier, position) in writes) + { + events[step - 1] = (position, 4096); + var count = cut == 0 || barrier < cut || (barrier == cut && completedBarrier) + ? 4096 : step == cut ? persistedPrefix : 0; + Array.Copy(completed.GetBuffer(), position, expectedImage, position, count); + } + + Assert.Equal(events.Take(cut == 0 ? events.Length : cut), device.Events); + var crashImage = device.CrashImage; + Assert.True(expectedImage.AsSpan().SequenceEqual(crashImage), $"Unexpected stable bytes at event {cut}"); + using var retry = JournalScenario.Copy(crashImage); + var analysis = NtfsJournal.Analyze(retry); + Assert.True(analysis.Status == expected, $"cut {cut}: {analysis}"); + + if (expected == NtfsJournalStatus.Unsupported) + { + AssertReplayRefused(retry, "damaged"); + return; + } + + if (cut == 3 && persistedPrefix == 4096) + { + // The primary page LSN advanced, while a different page never made it to storage. + Assert.Equal(1, analysis.RecordsToRedo); + } + + var storage = LogFileStorage.Open(retry, writer.Layout); + if (cut == 6 && persistedPrefix == 512) + { + // The old alternate's USN was the value the source would normally increment to. + Assert.Null(LfsRestartPage.TryRead(storage, TestJournalWriter.PageSize)); + Assert.True(LfsRestartPage.TryRead(storage, 0).RequiresRecovery); + } + + var result = NtfsJournal.Replay(retry); + Assert.Equal(expected != NtfsJournalStatus.Clean, result.Replayed); + var mftBytes = Read(retry, mft, 1024, logical: true); + Assert.Equal(updateLsn, U64(mftBytes, 8)); + Assert.Equal(Pattern.AsSpan(0, 8).ToArray(), mftBytes.AsSpan(attribute + 0x18, 8).ToArray()); + Assert.Equal(Read(retry, mft, 4096), Read(retry, mirror, 4096)); + Assert.Equal(Enumerable.Repeat(Pattern[0], 1024).ToArray(), Read(retry, rawLcn * 4096, 1024)); + var bytes = retry.ToArray(); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(retry).Status); + Assert.False(NtfsJournal.Replay(retry).Replayed); + Assert.Equal(bytes, retry.ToArray()); + } + + [Theory] + [InlineData(7)] + [InlineData(9)] + public void RetryOnTheSameStreamAfterAFailedRestartBarrier(int cut) + { + using var dirty = Volume(); + var (writer, _, _, _) = WriteRedoInterruptionFixture(dirty); + using var device = new CrashImageStream(dirty.ToArray(), cut); + Assert.Same(device.Failure, Assert.Throws(() => NtfsJournal.Replay(device))); + + using var stable = JournalScenario.Copy(device.CrashImage); + Assert.Equal(NtfsJournalStatus.NothingToReplay, NtfsJournal.Analyze(stable).Status); + var result = NtfsJournal.Replay(device); + if (cut == 7) + { + // The visible alternate may be buffered. The unconditional barrier runs before replacing the source. + Assert.True(result.Replayed); + using var fs = new NtfsFileSystem(dirty, NtfsJournalMode.Ignore); + var logStart = fs.PathToClusters("$LogFile").First().Offset * 4096; + Assert.Equal(new[] { (0L, 0), (logStart, 4096), (0L, 0) }, device.Events.Skip(cut)); + using var reopened = JournalScenario.Copy(device.CrashImage); + var storage = LogFileStorage.Open(reopened, writer.Layout); + Assert.True(LfsRestartPage.TryRead(storage, 0).IsClean); + Assert.True(LfsRestartPage.TryRead(storage, TestJournalWriter.PageSize).IsClean); + } + else + { + // Both pages are visible, so replay is a no-op. The stable alternate already protects the journal. + Assert.Equal(NtfsJournalStatus.Clean, result.Status); + Assert.False(result.Replayed); + Assert.Equal(cut, device.Events.Count); + Assert.True(NtfsJournal.Replay(stable).Replayed); + } + } + + [Fact] + public void AnotherInterruptionCanLeaveOneValidCleanRestartPage() + { + using var dirty = Volume(); + var (writer, _, _, _) = WriteRedoInterruptionFixture(dirty, sourceIsSecond: true); + Assert.Equal(TestJournalWriter.PageSize, LfsLog.Open(writer.Storage, out _).Restart.Offset); + + // The alternate reset is durable, while the source on the second page is still dirty. + using var first = new CrashImageStream(dirty.ToArray(), cut: 7, completedBarrier: true); + Assert.Same(first.Failure, Assert.Throws(() => NtfsJournal.Replay(first))); + using var second = new CrashImageStream(first.CrashImage, cut: 2, persistedPrefix: 512); + var analysis = NtfsJournal.Analyze(second); + Assert.Equal(NtfsJournalStatus.NothingToReplay, analysis.Status); + Assert.Contains("one restart page", analysis.Detail); + Assert.Throws(() => new NtfsFileSystem(second)); + Assert.Same(second.Failure, Assert.Throws(() => NtfsJournal.Replay(second))); + + using var retry = JournalScenario.Copy(second.CrashImage); + var storage = LogFileStorage.Open(retry, writer.Layout); + Assert.True(LfsRestartPage.TryRead(storage, 0).IsClean); + Assert.Null(LfsRestartPage.TryRead(storage, TestJournalWriter.PageSize)); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Replay(retry).Status); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AFlushThatDropsMetadataCanPublishAnUnsafeReset(bool publishBoth) + { + using var dirty = Volume(); + var (_, rawLcn, _, _) = WriteRedoInterruptionFixture(dirty); + var rawPosition = rawLcn * 4096; + using var device = new CrashImageStream(dirty.ToArray(), cut: publishBoth ? 0 : 7, completedBarrier: true, + persistWrite: position => position != rawPosition); + if (publishBoth) + { + Assert.True(NtfsJournal.Replay(device).Replayed); + } + else + { + Assert.Same(device.Failure, Assert.Throws(() => NtfsJournal.Replay(device))); + } + + using var retry = JournalScenario.Copy(device.CrashImage); + Assert.Equal(Read(dirty, rawPosition, 4096), Read(retry, rawPosition, 4096)); + if (publishBoth) + { + // After publishing both resets, no dirty restart page remains to identify the lost metadata. + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(retry).Status); + var before = retry.ToArray(); + Assert.False(NtfsJournal.Replay(retry).Replayed); + Assert.Equal(before, retry.ToArray()); + } + else + { + AssertReplayRefused(retry, "partial $LogFile reset disagrees"); + } + } + + [Theory] + [InlineData(true, "disagree about recovery at the same LSN")] + [InlineData(false, "disagree about the checkpoint at the same LSN")] + public void RestartPagesWithConflictingStateAtTheSameLsnAreRefused(bool isClean, string reason) + { + using var volume = Volume(); + var (writer, _, _, _) = WriteRedoInterruptionFixture(volume); + var conflict = writer.RestartPage(0, 0, clean: isClean, writer.LastLsn); + writer.Storage.Write(0, conflict, 0, conflict.Length); + AssertReplayRefused(volume, reason); + } + + private static (TestJournalWriter Writer, long RawLcn, ulong UpdateLsn, int Attribute) WriteRedoInterruptionFixture( + MemoryStream volume, bool sourceIsSecond = false) + { + var writer = new TestJournalWriter(volume, sequence: 1000); + var record = Read(volume, writer.Layout.MftCluster * 4096, 1024, logical: true); + var attribute = U16(record, 0x14); + var target = FreeCluster(volume); + var restart = writer.Checkpoint(Attributes); + var lsn = writer.Append(FirstMftValueUpdate(writer, record, Pattern.AsSpan(0, 8).ToArray(), new byte[8])); + writer.Append(Raw(target, Enumerable.Repeat(Pattern[0], 1024).ToArray())); + writer.Append(Forget()); + var currentLsn = sourceIsSecond ? writer.LastLsn - 2 : writer.LastLsn; + writer.Finish(restart, currentLsn: currentLsn); + + var other = writer.RestartPage(restart, 0, clean: false, sourceIsSecond ? currentLsn - 1 : currentLsn); + Assert.True(RemoveFixups(other, 0, other.Length)); + ApplyFixups(other, 0, other.Length); + writer.Storage.Write(sourceIsSecond ? 0 : TestJournalWriter.PageSize, other, 0, other.Length); + return (writer, target, lsn, attribute); + } + + [Fact] + public void ATornFileImageWithAnAdvancedLsnMustStillBeRedone() + { + using var dirty = Volume(); + var writer = new TestJournalWriter(dirty, sequence: 1000); + var mft = writer.Layout.MftCluster * 4096; + var original = Read(dirty, mft, 1024); + var image = (byte[])original.Clone(); + Assert.True(RemoveFixups(image, 0, image.Length)); + var usa = U16(image, 4); + var oldUsn = U16(image, usa); + // Incrementing the logged image's USA alone would reuse the destination's existing value. + Put16(image, usa, (ushort)(oldUsn - 1)); + var attr = U16(image, 0x14); + Array.Copy(Pattern, 0, image, attr + 0x18, 8); + var restart = writer.Checkpoint(Attributes); + var lsn = writer.Append(Op(NtfsLogOperation.InitializeFileRecordSegment, image, Mft, 0, [writer.Layout.MftCluster])); + writer.Append(Forget()); + writer.Finish(restart); + + using var device = new CrashImageStream(dirty.ToArray(), cut: 3, persistedPrefix: 512); + Assert.Same(device.Failure, Assert.Throws(() => NtfsJournal.Replay(device))); + using var retry = JournalScenario.Copy(device.CrashImage); + var torn = Read(retry, mft, original.Length); + Assert.Equal(lsn, U64(torn, 8)); + Assert.False(RemoveFixups(torn, 0, torn.Length)); + + Assert.Equal(1, NtfsJournal.Replay(retry).RecordsToRedo); + var result = Read(retry, mft, original.Length, logical: true); + Assert.Equal(Pattern.AsSpan(0, 8).ToArray(), result.AsSpan(attr + 0x18, 8).ToArray()); + } + + [Fact] + public void ATornFileRecordNeedsAReplacementOfAllUsedBytes() + { + using var volume = Volume(); + var layout = JournalVolumeLayout.Read(volume); + var position = layout.MftCluster * layout.BytesPerCluster; + var record = Read(volume, position, layout.MftRecordSize, logical: true); + volume.Position = position + 0x3FE; + var previous = volume.ReadByte(); + volume.Position--; + volume.WriteByte((byte)(previous ^ 0xFF)); + using var dirty = WithRecordsOn(volume, Op(NtfsLogOperation.InitializeFileRecordSegment, record.AsSpan(0, 0x60).ToArray(), + Mft, 0, [layout.MftCluster]), Forget()); + AssertReplayRefused(dirty, "complete replacement for the torn file record"); + } + + [Fact] + public void ALoggedIndexImageCannotOverwriteAnIndexWithALaterLsn() + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var lcn = FirstIndexBuffer(volume); + var position = lcn * 4096; + var index = Read(volume, position, 4096, logical: true); + var restart = writer.Checkpoint(Attributes); + writer.Append(Op(NtfsLogOperation.UpdateNonresidentValue, index, Index, 0, [lcn])); + writer.Append(Forget()); + writer.Finish(restart); + + index[0x100] ^= 0x45; + Put64(index, 8, writer.LastLsn); + ApplyFixups(index, 0, index.Length); + volume.Position = position; + volume.Write(index, 0, index.Length); + + Assert.Equal(NtfsJournalStatus.NothingToReplay, NtfsJournal.Replay(volume).Status); + Assert.Equal(index, Read(volume, position, index.Length)); + } + + [Theory] + [InlineData(false, "partially replaces a torn index buffer")] + [InlineData(true, "invalid replacement for the torn index buffer")] + public void ATornIndexBufferNeedsAValidCompleteReplacement(bool fullLength, string reason) + { + using var volume = Volume(); + var lcn = FirstIndexBuffer(volume); + volume.Position = lcn * 4096 + 0x3FE; + var previous = volume.ReadByte(); + volume.Position--; + volume.WriteByte((byte)(previous ^ 0xFF)); + using var dirty = WithRecordsOn(volume, Op(NtfsLogOperation.UpdateNonresidentValue, + fullLength ? new byte[4096] : Pattern, Index, 0, [lcn], recordOffset: fullLength ? 0 : 0x100), Forget()); + AssertReplayRefused(dirty, reason); + } + + // MemoryStream supplies the live read/write view only. This fixture explicitly models a second image + // that receives pending writes at successful barriers and can retain a prefix of the failing write. + private sealed class CrashImageStream : MemoryStream + { + private readonly byte[] _stable; + private readonly List<(int Position, int Count)> _pending = []; + private readonly int _cut; + private readonly int _persistedPrefix; + private readonly bool _completedBarrier; + private readonly Func? _persistWrite; + + public CrashImageStream(byte[] image, int cut, int persistedPrefix = 0, bool completedBarrier = false, + Func? persistWrite = null) + : base((byte[])image.Clone(), 0, image.Length, writable: true, publiclyVisible: true) + { + _stable = (byte[])image.Clone(); + _cut = cut; + _persistedPrefix = persistedPrefix; + _completedBarrier = completedBarrier; + _persistWrite = persistWrite; + } + + public IOException Failure { get; } = new("Injected interruption"); + + public List<(long Position, int Count)> Events { get; } = []; + + public byte[] CrashImage => (byte[])_stable.Clone(); + + public override void Write(byte[] buffer, int offset, int count) + { + var position = checked((int)Position); + Events.Add((position, count)); + if (Events.Count == _cut) + { + Array.Copy(buffer, offset, _stable, position, Math.Min(count, _persistedPrefix)); + throw Failure; + } + + base.Write(buffer, offset, count); + _pending.Add((position, count)); + } + + public override void Flush() + { + Events.Add(default); + var cut = Events.Count == _cut; + if (!cut || _completedBarrier) + { + var live = GetBuffer(); + foreach (var (position, count) in _pending) + { + if (_persistWrite == null || _persistWrite(position)) + { + Array.Copy(live, position, _stable, position, count); + } + } + + _pending.Clear(); + } + + if (cut) + { + throw Failure; + } + } + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalOpenModeTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalOpenModeTest.cs new file mode 100644 index 000000000..a3b587f5a --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalOpenModeTest.cs @@ -0,0 +1,160 @@ +using System; +using System.IO; +using System.Linq; +using DiscUtils; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Setup; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Default and explicit journal modes through the NTFS constructor and the file system manager. +/// +public class NtfsJournalOpenModeTest +{ + private static readonly Lazy Scenario = new(() => JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"dir\sub"))); + + /// A volume whose journal holds the creation of "dir", which is not on disk yet. + private static MemoryStream CrashedVolume() + { + var volume = JournalScenario.Copy(Scenario.Value.Before); + Scenario.Value.WriteJournal(volume, committed: true); + return volume; + } + + private static DiscFileSystem OpenThroughTheManager(Stream volume, FileSystemParameters? parameters = null) + { + SetupHelper.RegisterAssembly(typeof(NtfsFileSystem).Assembly); + var info = FileSystemManager.DetectFileSystems(volume).Single(f => f.Name == "NTFS"); + return parameters == null ? info.Open(volume) : info.Open(volume, parameters); + } + + [Theory] + [InlineData(" rEpLaY ", NtfsJournalMode.Replay)] + [InlineData("Refuse", NtfsJournalMode.Refuse)] + [InlineData("ignore", NtfsJournalMode.Ignore)] + public void TheNtfsJournalModeParameter_ChoosesTheModeForOneVolume(string mode, NtfsJournalMode expected) + { + using var volume = CrashedVolume(); + var image = volume.ToArray(); + var parameters = new FileSystemParameters { Parameters = { { "NtfsJournalMode", mode } } }; + + if (expected == NtfsJournalMode.Refuse) + { + Assert.Throws(() => OpenThroughTheManager(volume, parameters)); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + return; + } + + using var ntfs = (NtfsFileSystem)OpenThroughTheManager(volume, parameters); + if (expected == NtfsJournalMode.Ignore) + { + Assert.Null(ntfs.JournalInfo); + Assert.False(ntfs.DirectoryExists("dir")); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + else + { + Assert.True(ntfs.JournalInfo.Replayed); + Assert.False(ntfs.JournalInfo.ReplayedInMemory); + Assert.True(ntfs.DirectoryExists(@"dir\sub")); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void DefaultWritableOpen_RequiresACleanJournal(bool alreadyApplied) + { + var scenario = Scenario.Value; + using var volume = JournalScenario.Copy(scenario.Before); + if (alreadyApplied) + { + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint([]); + writer.Finish(restart); + } + else + { + scenario.WriteJournal(volume, committed: true); + } + var image = volume.ToArray(); + var expected = alreadyApplied ? NtfsJournalStatus.NothingToReplay : NtfsJournalStatus.NeedsReplay; + Assert.Equal(expected, NtfsJournal.Analyze(volume).Status); + + Assert.Throws(() => new NtfsFileSystem(volume)); + Assert.Throws(() => OpenThroughTheManager(volume)); + Assert.Throws(() => OpenThroughTheManager(volume, new FileSystemParameters())); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void DefaultWritableOpen_LeavesACleanVolumeUnchanged() + { + using var volume = JournalScenario.Copy(Scenario.Value.Before); + using (var fs = (NtfsFileSystem)OpenThroughTheManager(volume)) + { + Assert.Equal(NtfsJournalStatus.Clean, fs.JournalInfo.Status); + Assert.True(fs.CanWrite); + Assert.False(fs.JournalInfo.Replayed); + Assert.False(fs.JournalInfo.ReplayedInMemory); + } + + Assert.True(Scenario.Value.Before.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void DefaultReadOnlyOpenThroughTheManager_UsesAnOverlay() + { + using var writable = CrashedVolume(); + var image = writable.ToArray(); + using var volume = new MemoryStream(image, writable: false); + using var fs = (NtfsFileSystem)OpenThroughTheManager(volume); + Assert.Equal(NtfsJournalStatus.NeedsReplay, fs.JournalInfo.Status); + Assert.False(fs.JournalInfo.Replayed); + Assert.True(fs.JournalInfo.ReplayedInMemory); + Assert.False(fs.CanWrite); + Assert.True(fs.DirectoryExists(@"dir\sub")); + Assert.True(image.AsSpan().SequenceEqual(writable.ToArray())); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("no such mode")] + [InlineData("0")] + [InlineData("99")] + [InlineData("Replay, Refuse")] + public void InvalidModeParameter_IsRejectedBeforeWriting(string? mode) + { + using var volume = CrashedVolume(); + var image = volume.ToArray(); + var parameters = new FileSystemParameters { Parameters = { { "NtfsJournalMode", mode } } }; + Assert.Throws(() => OpenThroughTheManager(volume, parameters)); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void InvalidEnumMode_IsRejectedBeforeWriting() + { + using var volume = CrashedVolume(); + var image = volume.ToArray(); + Assert.Throws(() => new NtfsFileSystem(volume, (NtfsJournalMode)99)); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void JournalApis_RejectInvalidStreamArguments() + { + Assert.Throws(() => new NtfsFileSystem(null!)); + Assert.Throws(() => NtfsJournal.Analyze(null!)); + Assert.Throws(() => NtfsJournal.Replay(null!)); + + using var disposed = new MemoryStream(); + disposed.Dispose(); + Assert.Throws(() => NtfsJournal.Analyze(disposed)); + using var readOnly = new MemoryStream(Scenario.Value.Before, writable: false); + Assert.Throws(() => NtfsJournal.Replay(readOnly)); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalReplayDetailTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalReplayDetailTest.cs new file mode 100644 index 000000000..896919ee8 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalReplayDetailTest.cs @@ -0,0 +1,825 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using DiscUtils; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// The parts of a replay the page-image scenarios do not reach: real operations on index buffers and file records +/// through the whole replay, the checkpoint's tables, compensation records and top-level actions, freed and moved +/// clusters, both table layouts, tail copies, the $MFTMirr copy, and every way a journal is refused. +/// +public class NtfsJournalReplayDetailTest +{ + private const ushort Mft = 0x18; + private const ushort Index = 0x40; + private const ushort Data = 0x68; + private const uint Tx = 0x40; + + private static readonly (ushort, uint, ulong, uint, string)[] Attributes = + [ + (Mft, 0x80, 0x0001000000000000UL, 0, ""), + (Index, 0xA0, 0x0005000000000005UL, 4096, "$I30"), + (Data, 0x80, 0x0001000000000006UL, 0, ""), + ]; + + #region volume helpers + + private static MemoryStream Volume(int files = 150) + { + var stream = new MemoryStream(); + var geometry = Geometry.FromCapacity(16 * 1024 * 1024); + using (var ntfs = NtfsFileSystem.Format(stream, "DETAIL", geometry, 0, geometry.TotalSectorsLong)) + { + ntfs.CreateDirectory("big"); + for (var i = 0; i < files; i++) + { + using var s = ntfs.OpenFile($@"big\file-with-a-long-name-{i:D4}.txt", FileMode.Create); + s.WriteByte((byte)i); + } + } + + stream.Position = 0; + return stream; + } + + private static NtfsFileSystem Open(Stream volume) => new(volume, NtfsJournalMode.Ignore); + + /// Where an MFT record lives: its cluster, its offset there in 512-byte units, and its VCN in $MFT. + private static (long Lcn, int BlockOffset, long Vcn) MftRecord(Stream volume, long record) + { + using var ntfs = Open(volume); + var position = record * 1024; + var vcn = position / 4096; + long v = 0; + foreach (var range in ntfs.PathToClusters("$MFT")) + { + if (vcn < v + range.Count) + { + return (range.Offset + (vcn - v), (int)(position % 4096) / 512, vcn); + } + + v += range.Count; + } + + throw new InvalidOperationException("no such record"); + } + + private static long RecordOf(Stream volume, string path) + { + using var ntfs = Open(volume); + return ntfs.GetFileId(path) & 0x0000FFFFFFFFFFFF; + } + + private static long FirstIndexBuffer(Stream volume) + { + var bytes = ((MemoryStream)volume).ToArray(); + for (var lcn = 16; lcn < bytes.Length / 4096; lcn++) + { + if (EndianUtilities.ToUInt32LittleEndian(bytes, lcn * 4096) == JournalUtilities.SignatureIndx) + { + return lcn; + } + } + + throw new InvalidOperationException("no index buffer"); + } + + private static long FreeCluster(Stream volume) + { + var bytes = ((MemoryStream)volume).ToArray(); + for (var lcn = (bytes.Length / 4096) - 2; lcn > 16; lcn--) + { + if (bytes.AsSpan(lcn * 4096, 4096).ToArray().All(b => b == 0)) + { + return lcn; + } + } + + throw new InvalidOperationException("no free cluster"); + } + + private static byte[] ReadLogical(Stream volume, long position, int length) + { + var bytes = new byte[length]; + JournalUtilities.ReadExactly(volume, position, bytes, 0, length); + Assert.True(JournalUtilities.RemoveFixups(bytes, 0, length)); + return bytes; + } + + private static void WriteLogical(Stream volume, long position, byte[] logical) + { + var bytes = (byte[])logical.Clone(); + JournalUtilities.ApplyFixups(bytes, 0, bytes.Length); + volume.Position = position; + volume.Write(bytes, 0, bytes.Length); + } + + private static byte[] ReadRaw(Stream volume, long position, int length) + { + var bytes = new byte[length]; + JournalUtilities.ReadExactly(volume, position, bytes, 0, length); + return bytes; + } + + private static TestLogRecord RawWrite(long lcn, byte[] redo, byte[] undo, ushort attribute = Data) => new() + { + Redo = NtfsLogOperation.UpdateNonresidentValue, + Undo = NtfsLogOperation.UpdateNonresidentValue, + TransactionId = Tx, + RedoData = redo, + UndoData = undo, + TargetAttribute = attribute, + TargetVcn = lcn, + Lcns = [lcn], + }; + + private static TestLogRecord Forget(uint tx = Tx) => + new() { Redo = NtfsLogOperation.ForgetTransaction, Undo = NtfsLogOperation.CompensationLogRecord, TransactionId = tx }; + + private static void AssertRefused(Stream volume, string reason) + { + var image = ((MemoryStream)volume).ToArray(); + var info = NtfsJournal.Analyze(volume); + Assert.True(info.Status == NtfsJournalStatus.Unsupported, $"expected Unsupported ({reason}), got {info}"); + Assert.True(info.Detail.Contains(reason), $"expected '{reason}' in '{info.Detail}'"); + Assert.Throws(() => NtfsJournal.Replay(volume)); + Assert.True(image.AsSpan().SequenceEqual(((MemoryStream)volume).ToArray())); + } + + #endregion + + [Fact] + public void IndexBufferOperations_AreRedone_AndUndone() + { + foreach (var committed in new[] { true, false }) + { + using var volume = Volume(); + var lcn = FirstIndexBuffer(volume); + var original = ReadLogical(volume, lcn * 4096, 4096); + var entry = 0x18 + (int)EndianUtilities.ToUInt32LittleEndian(original, 0x18); + var oldInfo = original.AsSpan(entry + 0x18, 0x38).ToArray(); + var newInfo = Enumerable.Repeat((byte)0x77, 0x38).ToArray(); + + if (!committed) + { + // The change reached the disk, the transaction did not finish. + var changed = (byte[])original.Clone(); + newInfo.CopyTo(changed, entry + 0x18); + WriteLogical(volume, lcn * 4096, changed); + } + + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + var lsn = writer.Append(new TestLogRecord + { + Redo = NtfsLogOperation.UpdateFileNameAllocation, + Undo = NtfsLogOperation.UpdateFileNameAllocation, + TransactionId = Tx, + RedoData = newInfo, + UndoData = oldInfo, + TargetAttribute = Index, + TargetVcn = 0, + AttributeOffset = entry, + Lcns = [lcn], + }); + if (committed) + { + writer.Append(Forget()); + } + + writer.Finish(restart); + Stream? overlay = null; + NtfsJournalInfo info; + if (committed) + { + info = NtfsJournal.Replay(volume); + } + else + { + info = NtfsJournal.ReplayInMemory(volume, out var replayed); + overlay = replayed; + Assert.NotNull(overlay); + } + + using var overlayLifetime = overlay; + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + + var result = ReadLogical(overlay ?? volume, lcn * 4096, 4096); + Assert.True(result.AsSpan(entry + 0x18, 0x38).SequenceEqual(committed ? newInfo : oldInfo)); + Assert.Equal(lsn, EndianUtilities.ToUInt64LittleEndian(result, 8)); + Assert.Equal(committed ? 0 : 1, info.TransactionsToUndo); + } + } + + [Fact] + public void FileRecordOperations_AreRedone_AndUndone_AndAnUpToDateRecordIsLeftAlone() + { + using var volume = Volume(); + var record = RecordOf(volume, @"big\file-with-a-long-name-0007.txt"); + var (lcn, block, vcn) = MftRecord(volume, record); + var position = (lcn * 4096) + (block * 512); + var original = ReadLogical(volume, position, 1024); + var created = original.AsSpan(0x38 + 0x18, 8).ToArray(); + var newTime = BitConverter.GetBytes(0x01D0000000000000L); + + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + var update = new TestLogRecord + { + Redo = NtfsLogOperation.UpdateResidentValue, + Undo = NtfsLogOperation.UpdateResidentValue, + TransactionId = Tx, + RedoData = newTime, + UndoData = created, + TargetAttribute = Mft, + TargetVcn = vcn, + ClusterBlockOffset = block, + RecordOffset = 0x38, + AttributeOffset = 0x18, + Lcns = [lcn], + }; + var lsn = writer.Append(update); + writer.Append(Forget()); + writer.Finish(restart); + + var info = NtfsJournal.Replay(volume); + Assert.Contains(record, info.FileRecordsChanged); + var result = ReadLogical(volume, position, 1024); + Assert.True(result.AsSpan(0x38 + 0x18, 8).SequenceEqual(newTime)); + Assert.Equal(lsn, EndianUtilities.ToUInt64LittleEndian(result, 8)); + + // Now the record is newer than the log: a second journal with the same change is not needed. + var again = new TestJournalWriter(volume); + var restart2 = again.Checkpoint(Attributes); + update.RedoData = BitConverter.GetBytes(0x01C0000000000000L); + again.Append(update); + again.Append(Forget()); + again.Finish(restart2); + Assert.Equal(NtfsJournalStatus.NothingToReplay, NtfsJournal.Analyze(volume).Status); + } + + [Fact] + public void CheckpointTables_DirtyPagesAndTransactions_DriveRedoAndUndo() + { + foreach (var version in new uint[] { 0, 1 }) + { + // Entries are 0x2C bytes in version 0 and 0x28 in version 1: the third entry's offset differs. + var data = (ushort)(version == 0 ? 0x18 + 0x2C : Data); + using var volume = Volume(files: 5); + var lcn = FreeCluster(volume); + var before = ReadRaw(volume, lcn * 4096, 4096); + var after = Enumerable.Repeat((byte)0x5C, 4096).ToArray(); + volume.Position = lcn * 4096; + volume.Write(after, 0, after.Length); + + // The change was logged before the checkpoint; the checkpoint lists the page as dirty and the + // transaction as running. Nothing after the checkpoint finishes it. + var writer = new TestJournalWriter(volume); + var change = writer.Append(RawWrite(lcn, after, before, attribute: data)); + var restart = writer.Checkpoint( + [(Mft, 0x80, 0x0001000000000000UL, 0, ""), (data, 0x80, 0x0001000000000006UL, 0, "$X")], + [(data, lcn, change, [lcn])], + [(Tx, TransactionState.Active, change, change), (0x68, TransactionState.Committed, change, change)], + ntfsVersion: version); + writer.Finish(restart, oldestLsn: change); + + var info = NtfsJournal.ReplayInMemory(volume, out var overlay); + using var recovered = overlay; + Assert.True(info.Status == NtfsJournalStatus.NeedsReplay, $"v{version}: {info}"); + Assert.Equal(1, info.TransactionsToUndo); + Assert.True(before.AsSpan().SequenceEqual(ReadRaw(recovered, lcn * 4096, 4096))); + } + } + + [Fact] + public void CompensationRecords_AndTopLevelActions_LimitWhatIsUndone() + { + using var volume = Volume(files: 5); + var x = FreeCluster(volume); + var y = x - 1; + var z = x - 2; + var zeros = new byte[64]; + byte[] Fill(byte b) => Enumerable.Repeat(b, 64).ToArray(); + foreach (var lcn in new[] { x, y, z }) + { + volume.Position = lcn * 4096; + volume.Write(Fill(0xEE), 0, 64); + } + + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + + // X: a change that will be undone. Y: a nested action that ended (top level action): kept. + // Z: changed and then compensated already (the CLR's redo is the undo): stays as compensated. + var a = writer.Append(RawWrite(x, Fill(0xEE), zeros)); + var b = writer.Append(new TestLogRecord { Redo = NtfsLogOperation.UpdateNonresidentValue, Undo = NtfsLogOperation.UpdateNonresidentValue, TransactionId = Tx, RedoData = Fill(0xEE), UndoData = zeros, TargetAttribute = Data, TargetVcn = y, Lcns = [y], UndoNextLsn = a }); + var end = writer.Append(new TestLogRecord { Redo = NtfsLogOperation.EndTopLevelAction, Undo = NtfsLogOperation.Noop, TransactionId = Tx, UndoNextLsn = a }); + var c = writer.Append(new TestLogRecord { Redo = NtfsLogOperation.UpdateNonresidentValue, Undo = NtfsLogOperation.UpdateNonresidentValue, TransactionId = Tx, RedoData = Fill(0xEE), UndoData = zeros, TargetAttribute = Data, TargetVcn = z, Lcns = [z], UndoNextLsn = end }); + writer.Append(new TestLogRecord { Redo = NtfsLogOperation.UpdateNonresidentValue, Undo = NtfsLogOperation.CompensationLogRecord, TransactionId = Tx, RedoData = Fill(0x11), TargetAttribute = Data, TargetVcn = z, Lcns = [z], UndoNextLsn = a }); + Assert.True(b > a && c > b); + writer.Finish(restart); + + NtfsJournal.ReplayInMemory(volume, out var overlay); + using var recovered = overlay; + Assert.True(ReadRaw(recovered, x * 4096, 64).SequenceEqual(zeros)); + Assert.True(ReadRaw(recovered, y * 4096, 64).SequenceEqual(Fill(0xEE))); + Assert.True(ReadRaw(recovered, z * 4096, 64).SequenceEqual(Fill(0x11))); + } + + [Fact] + public void FreedClusters_AreNotReplayed_AndHotFixMovesAPage() + { + using var volume = Volume(files: 5); + var freed = FreeCluster(volume); + var original = freed - 1; + var moved = freed - 2; + var data = Enumerable.Repeat((byte)0x42, 128).ToArray(); + + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(RawWrite(freed, data, new byte[128])); + var range = new byte[16]; + EndianUtilities.WriteBytesLittleEndian((ulong)freed, range, 0); + EndianUtilities.WriteBytesLittleEndian(1UL, range, 8); + writer.Append(new TestLogRecord { Redo = NtfsLogOperation.DeleteDirtyClusters, Undo = NtfsLogOperation.Noop, TransactionId = Tx, RedoData = range }); + + // A page that moved (bad cluster replaced): the replay writes to the new cluster. + writer.Append(new TestLogRecord { Redo = NtfsLogOperation.UpdateNonresidentValue, Undo = NtfsLogOperation.UpdateNonresidentValue, TransactionId = Tx, RedoData = data, UndoData = new byte[128], TargetAttribute = Data, TargetVcn = 7000, Lcns = [original] }); + writer.Append(new TestLogRecord { Redo = NtfsLogOperation.HotFix, Undo = NtfsLogOperation.Noop, TransactionId = Tx, TargetAttribute = Data, TargetVcn = 7000, Lcns = [moved] }); + writer.Append(Forget()); + writer.Finish(restart); + + NtfsJournal.Replay(volume); + Assert.True(ReadRaw(volume, freed * 4096, 128).All(b => b == 0)); + Assert.True(ReadRaw(volume, original * 4096, 128).All(b => b == 0)); + Assert.True(ReadRaw(volume, moved * 4096, 128).SequenceEqual(data)); + } + + [Fact] + public void AnAttributeOpenedAfterTheCheckpoint_CanBeChanged() + { + foreach (var version in new uint[] { 0, 1 }) + { + using var volume = Volume(); + var lcn = FirstIndexBuffer(volume); + var original = ReadLogical(volume, lcn * 4096, 4096); + var changed = (byte[])original.Clone(); + changed[0x60] ^= 0xFF; + + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint([(Mft, 0x80, 0x0001000000000000UL, 0, "")], ntfsVersion: version); + writer.Append(new TestLogRecord + { + Redo = NtfsLogOperation.OpenNonresidentAttribute, + Undo = NtfsLogOperation.Noop, + TransactionId = Tx, + RedoData = TestJournalWriter.OpenAttributeEntryBytes(0xA0, 0x0005000000000005UL, 4096, version), + UndoData = Encoding.Unicode.GetBytes("$I30"), + TargetAttribute = 0x90, + }); + + // A whole new index buffer image (how NTFS logs a new one), in logical form. + writer.Append(RawWrite(lcn, changed, original, attribute: 0x90)); + writer.Append(Forget()); + writer.Finish(restart); + + NtfsJournal.Replay(volume); + Assert.True(ReadLogical(volume, lcn * 4096, 4096).AsSpan(0x40, 0x100).SequenceEqual(changed.AsSpan(0x40, 0x100))); + } + } + + [Fact] + public void BitmapOperations_SetAndClearRanges_AndAnEmptyRangeChangesNothing() + { + using var volume = Volume(files: 5); + var lcn = FreeCluster(volume); + byte[] Range(uint first, uint count) + { + var b = new byte[8]; + EndianUtilities.WriteBytesLittleEndian(first, b, 0); + EndianUtilities.WriteBytesLittleEndian(count, b, 4); + return b; + } + + TestLogRecord Bits(NtfsLogOperation op, uint first, uint count) => + new() { Redo = op, Undo = NtfsLogOperation.Noop, TransactionId = Tx, RedoData = Range(first, count), TargetAttribute = Data, TargetVcn = lcn, Lcns = [lcn], RecordOffset = 16 }; + + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(Bits(NtfsLogOperation.SetBitsInNonresidentBitMap, 3, 20)); + writer.Append(Bits(NtfsLogOperation.ClearBitsInNonresidentBitMap, 5, 2)); + writer.Append(Bits(NtfsLogOperation.SetBitsInNonresidentBitMap, 100, 0)); + writer.Append(Forget()); + writer.Finish(restart); + + NtfsJournal.Replay(volume); + Assert.Equal(new byte[] { 0x98, 0xFF, 0x7F, 0 }, ReadRaw(volume, (lcn * 4096) + 16, 4)); + } + + [Fact] + public void TailCopies_HoldTheNewestPages() + { + foreach (var version in new[] { 1, 2 }) + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"tail")); + using var volume = JournalScenario.Copy(scenario.Before); + var writer = new TestJournalWriter(volume, version) { TailCopies = true }; + var restart = writer.Checkpoint(Attributes); + ulong previous = 0; + foreach (var record in scenario.Records) + { + record.UndoNextLsn = previous; + previous = writer.Append(record); + } + + writer.Append(Forget(JournalScenario.Transaction)); + + // The page with the last records only made it to the tail area before the crash. + writer.PagesOnlyInTail.Add(writer.LsnToOffsetForTest(writer.LastLsn) & ~(long)(TestJournalWriter.PageSize - 1)); + writer.Finish(restart); + + NtfsJournal.Replay(volume); + scenario.AssertVolumeEquals(scenario.After, volume); + } + } + + [Fact] + public void RestartPages_TheNewerValidCopyIsUsed() + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"restart")); + foreach (var damageFirst in new[] { true, false }) + { + using var volume = JournalScenario.Copy(scenario.Before); + var writer = scenario.WriteJournal(volume, committed: true); + var page = new byte[TestJournalWriter.PageSize]; + if (damageFirst) + { + // A torn first copy: the second one is used. + writer.Storage.Read(0, page, 0, page.Length); + page[0x3FE] ^= 0xFF; + writer.Storage.Write(0, page, 0, page.Length); + } + else + { + // An older first copy (lower current LSN): the second one is used. + var older = writer.RestartPage(0, 0, clean: true, currentLsn: 1); + writer.Storage.Write(0, older, 0, older.Length); + } + + NtfsJournal.Replay(volume); + scenario.AssertVolumeEquals(scenario.After, volume); + } + } + + [Fact] + public void MftRecordsZeroToThree_AreCopiedToTheMftMirror() + { + // Enough new files to grow $MFT: record 0 ($MFT's own runs) changes. + var scenario = JournalScenario.Create(ntfs => + { + for (var i = 0; i < 400; i++) + { + ntfs.CreateDirectory($"d{i}"); + } + }); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + NtfsJournal.Replay(volume); + scenario.AssertVolumeEquals(scenario.After, volume); + + using var ntfs = Open(volume); + var mft = ntfs.PathToClusters("$MFT").First().Offset; + var mirror = ntfs.PathToClusters("$MFTMirr").First().Offset; + Assert.True(ReadRaw(volume, mft * 4096, 4096).SequenceEqual(ReadRaw(volume, mirror * 4096, 4096))); + } + + [Fact] + public void LogFileRecord_ComesFromTheMirrorWhenTheMftCopyIsDamaged() + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"mirror")); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + var (lcn, block, _) = MftRecord(volume, 2); + var position = (lcn * 4096) + (block * 512); + volume.Position = position + 0x1FE; + var original = volume.ReadByte(); + volume.Position = position + 0x1FE; + volume.WriteByte(0x99); + + NtfsJournal.Replay(volume); + volume.Position = position + 0x1FE; + volume.WriteByte((byte)original); + scenario.AssertVolumeEquals(scenario.After, volume); + + // Both copies damaged: the log cannot be found. + using var both = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(both, committed: true); + using (var ntfs = Open(both)) + { + foreach (var path in new[] { "$MFT", "$MFTMirr" }) + { + var start = (ntfs.PathToClusters(path).First().Offset * 4096) + (2 * 1024); + both.Position = start + 0x1FE; + both.WriteByte(0x99); + } + } + + AssertRefused(both, "$LogFile"); + } + + [Theory] + [InlineData("resident")] + [InlineData("no data")] + [InlineData("sparse")] + [InlineData("runs")] + public void LogFileRecord_ThatIsNotWhatNtfsWrites_IsRefused(string damage) + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"x")); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + using (var ntfs = Open(volume)) + { + foreach (var path in new[] { "$MFT", "$MFTMirr" }) + { + var start = (ntfs.PathToClusters(path).First().Offset * 4096) + (2 * 1024); + var record = ReadLogical(volume, start, 1024); + var attr = (int)EndianUtilities.ToUInt16LittleEndian(record, 0x14); + while (EndianUtilities.ToUInt32LittleEndian(record, attr) != 0x80) + { + attr += (int)EndianUtilities.ToUInt32LittleEndian(record, attr + 4); + } + + var runs = attr + EndianUtilities.ToUInt16LittleEndian(record, attr + 0x20); + switch (damage) + { + case "resident": record[attr + 8] = 0; break; + case "no data": EndianUtilities.WriteBytesLittleEndian(0x81u, record, attr); break; + case "sparse": record[runs] = 0x01; break; + case "runs": record[runs] = 0x99; break; + } + + WriteLogical(volume, start, record); + } + } + + AssertRefused(volume, "$LogFile"); + } + + [Fact] + public void Journals_ThatAreDamagedOrUnsupported_AreRefused_AndNothingIsWritten() + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"refuse")); + + MemoryStream With(Action damage, bool committed = true) + { + var volume = JournalScenario.Copy(scenario.Before); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + damage(writer, restart); + return volume; + } + + // No restart page at all, and not an empty (0xFF) log. + using (var volume = JournalScenario.Copy(scenario.Before)) + { + var writer = new TestJournalWriter(volume); + writer.Storage.Write(0, new byte[8192], 0, 8192); + AssertRefused(volume, "no valid restart page"); + } + + // A log file service version DiscUtils does not know. + using (var volume = With((w, r) => + { + w.Finish(r); + var page = new byte[TestJournalWriter.PageSize]; + w.Storage.Read(0, page, 0, page.Length); + Assert.True(JournalUtilities.RemoveFixups(page, 0, page.Length)); + EndianUtilities.WriteBytesLittleEndian((ushort)3, page, 0x1C); + JournalUtilities.ApplyFixups(page, 0, page.Length); + w.Storage.Write(0, page, 0, page.Length); + w.Storage.Write(TestJournalWriter.PageSize, page, 0, page.Length); + })) + { + AssertRefused(volume, "version 3"); + } + + // An implausible restart area. + using (var volume = With((w, r) => + { + w.Finish(r); + var page = new byte[TestJournalWriter.PageSize]; + w.Storage.Read(0, page, 0, page.Length); + Assert.True(JournalUtilities.RemoveFixups(page, 0, page.Length)); + EndianUtilities.WriteBytesLittleEndian(8u, page, 0x30 + 0x10); + JournalUtilities.ApplyFixups(page, 0, page.Length); + w.Storage.Write(0, page, 0, page.Length); + w.Storage.Write(TestJournalWriter.PageSize, page, 0, page.Length); + })) + { + AssertRefused(volume, "implausible restart area"); + } + + // The restart record the restart area points at is not in the log. + using (var volume = With((w, r) => w.Finish(r + 0x100))) + { + AssertRefused(volume, "restart record"); + } + + // A record without a transaction. + using (var volume = With((w, r) => + { + w.Append(new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0 }); + w.Finish(r); + })) + { + AssertRefused(volume, "no transaction"); + } + + // A record naming more clusters than its dirty page holds. + using (var volume = With((w, r) => + { + var record = RawWrite(100, new byte[8], new byte[8]); + record.Lcns = [100, 101]; + w.Append(record); + w.Finish(r); + })) + { + AssertRefused(volume, "beyond its dirty page"); + } + + // An undo chain that loops, points at another transaction, or at nothing. + foreach (var (link, reason) in new (Func, string)[] + { + (self => self, "loops"), + (self => self - 0x10, "belongs to"), + (self => self + 0x1000, "points forward"), + }) + { + using var volume = With((w, r) => + { + var other = w.Append(new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0x68 }); + var record = RawWrite(100, new byte[8], new byte[8]); + var lsn = w.LsnOf(w.LsnToOffsetForTest(other) + 0x50, 1); + record.UndoNextLsn = reason == "belongs to" ? other : link(lsn); + var actual = w.Append(record); + Assert.Equal(lsn, actual); + w.Finish(r); + }); + AssertRefused(volume, reason); + } + + // A change to an attribute that is not open. + using (var volume = With((w, r) => + { + w.Append(RawWrite(100, new byte[8], new byte[8], attribute: 0x200)); + w.Append(Forget()); + w.Finish(r); + })) + { + AssertRefused(volume, "not open"); + } + + // A bit range record that is too short. + using (var volume = With((w, r) => + { + w.Append(new TestLogRecord { Redo = NtfsLogOperation.SetBitsInNonresidentBitMap, Undo = NtfsLogOperation.Noop, TransactionId = Tx, RedoData = new byte[4], TargetAttribute = Data, TargetVcn = 100, Lcns = [100] }); + w.Append(Forget()); + w.Finish(r); + })) + { + AssertRefused(volume, "bit range"); + } + + // A whole index buffer image that does not fit the buffer. + using (var volume = With((w, r) => + { + w.Append(new TestLogRecord { Redo = NtfsLogOperation.UpdateNonresidentValue, Undo = NtfsLogOperation.Noop, TransactionId = Tx, RedoData = new byte[64], TargetAttribute = Index, TargetVcn = 100, RecordOffset = 4090, Lcns = [100] }); + w.Append(Forget()); + w.Finish(r); + })) + { + AssertRefused(volume, "past its index buffer"); + } + + // Clusters that are not on the volume. + using (var volume = With((w, r) => + { + w.Append(RawWrite(1L << 40, new byte[8], new byte[8])); + w.Append(Forget()); + w.Finish(r); + })) + { + AssertRefused(volume, "not on the volume"); + } + } + + [Fact] + public void DamagedPages_AreRefused_ExceptARecordThatIsRebuiltOrAHole() + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"pages")); + using var probe = JournalScenario.Copy(scenario.Before); + var lcn = FreeCluster(probe); + + MemoryStream With(byte[] page, TestLogRecord record) + { + var volume = JournalScenario.Copy(scenario.Before); + volume.Position = lcn * 4096; + volume.Write(page, 0, page.Length); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(record); + writer.Append(Forget()); + writer.Finish(restart); + return volume; + } + + TestLogRecord On(NtfsLogOperation op, ushort attribute, byte[] data, int recordOffset = 0, int attributeOffset = 0) => + new() { Redo = op, Undo = NtfsLogOperation.Noop, TransactionId = Tx, RedoData = data, TargetAttribute = attribute, TargetVcn = lcn, RecordOffset = recordOffset, AttributeOffset = attributeOffset, Lcns = [lcn] }; + + var zeros = new byte[4096]; + + // A change to a file record that is not one. + using (var volume = With(zeros, On(NtfsLogOperation.UpdateResidentValue, Mft, new byte[8], 0x38, 0x18))) + { + AssertRefused(volume, "damaged"); + } + + // A change to an index buffer that is not one. + using (var volume = With(zeros, On(NtfsLogOperation.UpdateFileNameAllocation, Index, new byte[0x38], 0, 0x40))) + { + AssertRefused(volume, "damaged"); + } + + // A page chkdsk marked bad. + var baad = new byte[4096]; + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureBaad, baad, 0); + using (var volume = With(baad, On(NtfsLogOperation.UpdateResidentValue, Mft, new byte[8], 0x38, 0x18))) + { + AssertRefused(volume, "bad"); + } + + // A hole is skipped. + var hole = new byte[4096]; + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureHole, hole, 0); + using (var volume = With(hole, On(NtfsLogOperation.UpdateResidentValue, Mft, new byte[8], 0x38, 0x18))) + { + Assert.Equal(NtfsJournalStatus.NothingToReplay, NtfsJournal.Analyze(volume).Status); + } + + // A file record that was never written is rebuilt from its InitializeFileRecordSegment. + var root = MftRecord(probe, 5); + var image = ReadLogical(probe, (root.Lcn * 4096) + (root.BlockOffset * 512), 1024); + using (var volume = With(zeros, On(NtfsLogOperation.InitializeFileRecordSegment, Mft, image.AsSpan(0, (int)EndianUtilities.ToUInt32LittleEndian(image, 0x18)).ToArray()))) + { + NtfsJournal.Replay(volume); + var rebuilt = ReadLogical(volume, lcn * 4096, 1024); + Assert.Equal(JournalUtilities.SignatureFile, EndianUtilities.ToUInt32LittleEndian(rebuilt, 0)); + } + } + + [Fact] + public void Replay_NeedsAWritableStream_AndInMemoryReplayOfACleanVolumeChangesNothing() + { + var scenario = JournalScenario.Create(ntfs => ntfs.CreateDirectory(@"stream")); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + using var readOnly = new MemoryStream(volume.ToArray(), writable: false); + Assert.Throws(() => NtfsJournal.Replay(readOnly)); + + using var clean = new MemoryStream(scenario.After, writable: false); + var info = NtfsJournal.ReplayInMemory(clean, out var replayed); + Assert.Equal(NtfsJournalStatus.Clean, info.Status); + Assert.Null(replayed); + Assert.Contains("Clean", info.ToString()); + } + + [Fact] + public void Fixups_AreChecked_AndTheSequenceNumberWraps() + { + var record = new byte[1024]; + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureFile, record, 0); + EndianUtilities.WriteBytesLittleEndian((ushort)0x30, record, 4); + EndianUtilities.WriteBytesLittleEndian((ushort)3, record, 6); + EndianUtilities.WriteBytesLittleEndian((ushort)0xFFFE, record, 0x30); + record[0x1FE] = 0xAB; + JournalUtilities.ApplyFixups(record, 0, record.Length); + Assert.Equal(1, EndianUtilities.ToUInt16LittleEndian(record, 0x30)); + Assert.Equal(1, EndianUtilities.ToUInt16LittleEndian(record, 0x1FE)); + + var torn = (byte[])record.Clone(); + torn[0x3FE] = 9; + Assert.False(JournalUtilities.RemoveFixups(torn, 0, torn.Length)); + Assert.True(JournalUtilities.RemoveFixups(record, 0, record.Length)); + Assert.Equal(0xAB, record[0x1FE]); + + Assert.Throws(() => JournalUtilities.ApplyFixups(new byte[1024], 0, 1024)); + Assert.False(JournalUtilities.HasFixupLayout(new byte[1000], 0, 1000)); + Assert.Throws(() => JournalUtilities.ReadExactly(new MemoryStream(new byte[10]), 0, new byte[20], 0, 20)); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalSafetyTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalSafetyTest.cs new file mode 100644 index 000000000..0b3a29079 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalSafetyTest.cs @@ -0,0 +1,737 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; +using File = System.IO.File; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace LibraryTests.Ntfs.Journal; + +public partial class NtfsJournalEdgeCaseTest +{ + private static void AssertReplayRefused(MemoryStream volume, string reason) + { + var before = volume.ToArray(); + AssertRefused(volume, reason); + Assert.Contains(reason, Assert.Throws(() => NtfsJournal.Replay(volume)).Message); + Assert.True(before.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Theory] + [InlineData("short header", "non-resident data header")] + [InlineData("attribute length overflow", "damaged attribute list")] + [InlineData("runs in header", "mapping pairs offset")] + [InlineData("runs past attribute", "mapping pairs offset")] + [InlineData("starting VCN", "nonzero starting VCN")] + [InlineData("last VCN", "declared allocation")] + [InlineData("attribute list after data", "$ATTRIBUTE_LIST")] + [InlineData("duplicate data", "split unnamed data")] + [InlineData("physical run outside volume", "outside the volume")] + [InlineData("physical run overlaps MFT", "overlaps $MFT")] + public void InvalidLogFileLayouts_AreRefusedBeforeReadingOrResettingTheLog(string damage, string reason) + { + using var volume = Volume(); + var layout = JournalVolumeLayout.Read(volume); + var position = layout.MftCluster * layout.BytesPerCluster + 2L * layout.MftRecordSize; + var record = Read(volume, position, layout.MftRecordSize, logical: true); + var data = (int)U16(record, 0x14); + while (U32(record, data) != 0x80) + { + data += (int)U32(record, data + 4); + } + + var attributeLength = (int)U32(record, data + 4); + var end = data + attributeLength; + Assert.Equal(0xFFFFFFFFu, U32(record, end)); + switch (damage) + { + case "short header": + Put32(record, data + 4, 0x38); + Put32(record, data + 0x38, 0xFFFFFFFF); + break; + case "attribute length overflow": + Put32(record, data + 4, int.MaxValue - 7); + break; + case "runs in header": + Put16(record, data + 0x20, 0x20); + break; + case "runs past attribute": + Put16(record, data + 0x20, ushort.MaxValue); + break; + case "starting VCN": + Put64(record, data + 0x10, 1); + break; + case "last VCN": + Put64(record, data + 0x18, U64(record, data + 0x18) + 1); + break; + case "attribute list after data": + Put32(record, end, 0x20); + Put32(record, end + 4, 0x18); + Put32(record, end + 0x18, 0xFFFFFFFF); + Put32(record, 0x18, (uint)(end + 0x20)); + break; + case "duplicate data": + Array.Copy(record, data, record, end, attributeLength); + Put32(record, end + attributeLength, 0xFFFFFFFF); + Put32(record, 0x18, (uint)(end + attributeLength + 8)); + break; + default: + var run = data + U16(record, data + 0x20); + record[run] = 0x22; + Put16(record, run + 1, (ushort)(U64(record, data + 0x18) + 1)); + Put16(record, run + 3, (ushort)(damage == "physical run overlaps MFT" ? layout.MftCluster : layout.TotalClusters - 1)); + record[run + 5] = 0; + // Give the backing stream room past the declared NTFS volume. + volume.SetLength(volume.Length + (1024 * 1024) * 4); + break; + } + + ApplyFixups(record, 0, record.Length); + volume.Position = position; + volume.Write(record, 0, record.Length); + AssertReplayRefused(volume, reason); + } + + [Fact] + public void ALogFileRunBeyondTheBackingStream_IsRefusedEvenWithAnIntactRestartArea() + { + using var volume = WithRecords(Forget()); + var layout = JournalVolumeLayout.Read(volume); + var record = Read(volume, layout.MftCluster * layout.BytesPerCluster + 2L * layout.MftRecordSize, + layout.MftRecordSize, logical: true); + var (runs, _) = LogFileStorage.ReadDataAttribute(record, layout, "$LogFile"); + var endLcn = runs.Max(run => run.Lcn + run.Length); + volume.SetLength((endLcn - 1) * layout.BytesPerCluster); + AssertReplayRefused(volume, "outside the volume"); + } + + public static IEnumerable InvalidLogRuns() + { + yield return [new byte[] { 0x11, 0, 10, 0 }, "invalid length"]; + yield return [new byte[] { 0x11, 1, 0, 0 }, "invalid or overflowing LCN"]; + yield return [new byte[] { 0x11, 1, 0xFF, 0 }, "invalid or overflowing LCN"]; + yield return [new byte[] { 0x18, 0, 0, 0, 0, 0, 0, 0, 0x80, 10, 0 }, "invalid length"]; + yield return [new byte[] { 0x81, 1, 0xFD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0x11, 1, 4, 0 }, "overflowing LCN"]; + yield return [new byte[] { 0x18, 0xFD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 1, 0x11, 4, 0, 0 }, "invalid length"]; + yield return [new byte[] { 0x11, 8, 0x20, 0x11, 4, 1, 0 }, "overlapping physical runs"]; + yield return [new byte[] { 0x11, 1, 0x20 }, "no terminator"]; + yield return [new byte[] { 0 }, "empty"]; + } + + [Theory] + [MemberData(nameof(InvalidLogRuns))] + public void MalformedMappingPairs_AreRefused(byte[] runs, string reason) + { + var error = Assert.Throws(() => LogFileStorage.ParseRuns(runs, 0, runs.Length, long.MaxValue, "$LogFile")); + Assert.Contains(reason, error.Message); + } + + [Theory] + [InlineData("boot cluster")] + [InlineData("overflowing LCN")] + [InlineData("overlaps MFT")] + [InlineData("redirected within volume")] + [InlineData("straddles volume end")] + public void InvalidMirrorDestinations_AreRefusedBeforeAnyWrites(string damage) + { + using var volume = NtfsJournalWindowsSampleTest.LoadVolume("s512"); + NtfsJournal.Replay(volume); + var layout = WriteFirstMftRecordUpdate(volume); + Assert.Equal(NtfsJournalStatus.NeedsReplay, NtfsJournal.Analyze(volume).Status); + + var boot = Read(volume, 0, 512); + var mirror = damage switch + { + "boot cluster" => 0, + "overflowing LCN" => (1L << 55) + layout.MftMirrorCluster, + "overlaps MFT" => layout.MftCluster + 1, + "redirected within volume" => layout.MftMirrorCluster + 8, + _ => layout.TotalClusters - 1, + }; + Put64(boot, 0x38, (ulong)mirror); + volume.Position = 0; + volume.Write(boot, 0, boot.Length); + AssertReplayRefused(volume, "$MFTMirr"); + } + + private static JournalVolumeLayout WriteFirstMftRecordUpdate(Stream volume) + { + var w = new TestJournalWriter(volume, sequence: 1000); + var layout = w.Layout; + var record = Read(volume, layout.MftCluster * layout.BytesPerCluster, layout.MftRecordSize, logical: true); + var count = (layout.MftRecordSize + layout.BytesPerCluster - 1) / layout.BytesPerCluster; + var lcns = Enumerable.Range(0, count).Select(i => layout.MftCluster + i).ToArray(); + var restart = w.Checkpoint(Attributes); + w.Append(Op(NtfsLogOperation.UpdateResidentValue, Pattern, Mft, 0, lcns, recordOffset: U16(record, 0x14), + attributeOffset: 0x18, undo: NtfsLogOperation.UpdateResidentValue, undoData: new byte[Pattern.Length])); + w.Append(Forget()); + w.Finish(restart); + return layout; + } + + [Fact] + public void MirroringAReplay_DoesNotOverwriteTheLogFileFallbackRecord() + { + using var volume = Volume(); + var layout = WriteFirstMftRecordUpdate(volume); + var offset = 2L * layout.MftRecordSize; + var primary = layout.MftCluster * layout.BytesPerCluster + offset; + var mirror = layout.MftMirrorCluster * layout.BytesPerCluster + offset; + var fallback = Read(volume, mirror, layout.MftRecordSize); + volume.Position = primary + 0x1FE; + var old = volume.ReadByte(); + volume.Position--; + volume.WriteByte((byte)(old ^ 0xFF)); + + Assert.Equal(NtfsJournalStatus.NeedsReplay, NtfsJournal.Replay(volume).Status); + Assert.Equal(fallback, Read(volume, mirror, fallback.Length)); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + } + + [Fact] + public void ReplayCannotWriteMetadataIntoLogFileClusters() + { + using var volume = Volume(); + var layout = JournalVolumeLayout.Read(volume); + var record = Read(volume, layout.MftCluster * layout.BytesPerCluster + 2L * layout.MftRecordSize, + layout.MftRecordSize, logical: true); + var (runs, _) = LogFileStorage.ReadDataAttribute(record, layout, "$LogFile"); + using var dirty = WithRecordsOn(volume, Raw(runs[0].Lcn, Pattern), Forget()); + AssertReplayRefused(dirty, "metadata replay write overlaps $LogFile"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ALogicalPageMustHaveTheSameCompleteClusterMapping(bool duplicateCluster) + { + using var volume = NtfsJournalWindowsSampleTest.LoadVolume("s512"); + NtfsJournal.Replay(volume); + var w = new TestJournalWriter(volume, sequence: 1000); + var restart = w.Checkpoint(Attributes); + var cluster = w.Layout.TotalClusters - 16; + w.Append(Op(NtfsLogOperation.InitializeFileRecordSegment, new byte[1024], Mft, 0, [cluster, cluster + 1])); + w.Append(Op(NtfsLogOperation.InitializeFileRecordSegment, new byte[1024], Mft, 8, + [cluster, duplicateCluster ? cluster : cluster + 2])); + w.Append(Forget()); + w.Finish(restart); + AssertReplayRefused(volume, duplicateCluster ? "more than one part of a page" : "conflicting cluster mappings"); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void UnknownOperationsWithNoClusters_AreRefused(bool beforeCheckpoint, bool undo) + { + using var volume = Volume(); + var w = new TestJournalWriter(volume); + var unknown = new TestLogRecord + { + Redo = undo ? NtfsLogOperation.Noop : (NtfsLogOperation)0xFFFF, + Undo = undo ? (NtfsLogOperation)0xFFFF : NtfsLogOperation.Noop, + TransactionId = 0x90, + }; + + ulong restart; + if (beforeCheckpoint) + { + var lsn = w.Append(unknown); + restart = w.Checkpoint(Attributes, transactions: [(0x90, TransactionState.Committed, lsn, lsn)]); + } + else + { + restart = w.Checkpoint(Attributes); + w.Append(unknown); + } + + w.Finish(restart); + AssertReplayRefused(volume, "unsupported redo or undo operation"); + } + + [Theory] + [InlineData("overflowing table size", "implausible header")] + [InlineData("short dirty page entry", "dirty page table entry is too short")] + [InlineData("overflowing cluster count", "more clusters than it holds")] + [InlineData("overflowing dirty page VCN", "invalid VCN range")] + [InlineData("short transaction entry", "transaction table entry is too short")] + [InlineData("negative table length", "invalid table length")] + public void CheckpointTables_AreBoundedBeforeReadingEntries(string damage, string reason) + { + using var volume = WithRestart((w, checkpoint) => + { + var table = Table(damage is "overflowing cluster count" or "overflowing dirty page VCN" ? 0x28 : 8, 1); + if (damage == "overflowing table size") + { + Put16(table, 0, ushort.MaxValue); + Put16(table, 2, ushort.MaxValue); + } + else if (damage == "overflowing cluster count") + { + Put32(table, 0x18 + 0x0C, 0x20000000); + } + else if (damage == "overflowing dirty page VCN") + { + Put32(table, 0x18 + 0x0C, 1); + Put64(table, 0x18 + 0x10, long.MaxValue); + } + + if (damage == "short transaction entry") + { + var restart = RestartRecord(1, checkpoint); + Put64(restart, 0x28, Dump(w, NtfsLogOperation.TransactionTableDump, table)); + Put32(restart, 0x3C, (uint)table.Length); + return restart; + } + + return RestartRecord(1, checkpoint, dirty: Dump(w, NtfsLogOperation.DirtyPageTableDump, table), + dirtyLength: damage == "negative table length" ? -1 : table.Length); + }); + AssertReplayRefused(volume, reason); + } + + [Fact] + public void AMissingRedoStart_CannotDiscardPendingChanges() + { + using var volume = Volume(); + var target = FreeCluster(volume); + var w = new TestJournalWriter(volume); + var missing = w.LsnOf(w.FirstPageOffset + 0x40, 0); + var restart = w.Checkpoint(Attributes, dirtyPages: [(Data, target, missing, [target])]); + w.Append(Raw(target, Pattern)); + w.Append(Forget()); + w.Finish(restart); + AssertReplayRefused(volume, "redo start record"); + } + + [Theory] + [InlineData(0x0D, 0)] + [InlineData(0x0D, 0xE0)] + [InlineData(0x40, 0xE1)] + [InlineData(0x44, 0xE1)] + public void InvalidBootGeometry_IsRefusedBeforeRecordAllocation(int offset, int value) + { + using var volume = Volume(); + volume.Position = offset; + volume.WriteByte((byte)value); + AssertReplayRefused(volume, "invalid volume or record sizes"); + } + + [Fact] + public void InvalidFixupRanges_AreRejectedWithoutIndexingTheBuffer() + { + Assert.False(RemoveFixups(new byte[7], 0, 7)); + Assert.False(RemoveFixups(new byte[512], 1, 512)); + var page = new byte[512]; + Put16(page, 4, 508); + Put16(page, 6, 2); + Assert.False(HasFixupLayout(page, 0, page.Length)); + } + + [Theory] + [InlineData(0x7FFFFFFFu)] + [InlineData(0xFFFFFFFFu)] + public void InvalidIndexBufferSizes_AreRefusedBeforePageAllocation(uint indexSize) + { + using var volume = Volume(); + var w = new TestJournalWriter(volume); + var restart = w.Checkpoint([(Index, 0xA0, 5, indexSize, "$I30")]); + w.Append(Op(NtfsLogOperation.UpdateNonresidentValue, Pattern, Index, 0, [FreeCluster(volume)])); + w.Append(Forget()); + w.Finish(restart); + AssertReplayRefused(volume, "invalid index buffer size"); + } + + [Fact] + public void AnOversizedBitmapOperation_IsRefusedBeforeAllocation() + { + using var probe = Volume(); + var range = new byte[8]; + Put32(range, 4, uint.MaxValue); + using var volume = WithRecords(Op(NtfsLogOperation.SetBitsInNonresidentBitMap, range, Data, 0, [FreeCluster(probe)]), Forget()); + AssertReplayRefused(volume, "oversized bit range"); + } + + [Theory] + [InlineData(-1L, "invalid target VCN")] + [InlineData(long.MaxValue, "invalid target VCN")] + [InlineData(long.MaxValue / 2, "invalid file record VCN")] + public void InvalidTargetVcns_AreRefusedBeforeReplay(long vcn, string reason) + { + using var probe = Volume(); + using var volume = WithRecords(Op(NtfsLogOperation.InitializeFileRecordSegment, new byte[1024], Mft, vcn, [FreeCluster(probe)]), Forget()); + AssertReplayRefused(volume, reason); + } + + [Fact] + public void ARangeOfFreedClustersMustNotOverflow() + { + var free = DeleteClusters(long.MaxValue); + using var volume = WithRecords(free, Forget()); + AssertReplayRefused(volume, "invalid range of freed clusters"); + } + + [Theory] + [InlineData(0x10, 60)] + [InlineData(0x18, 0x20000)] + [InlineData(0x26, 0xFF8)] + public void InvalidRestartGeometry_IsRefusedBeforeReplay(int field, int value) + { + using var volume = WithRecords(Forget()); + PatchRestartPages(new TestJournalWriter(volume), page => + { + if (field == 0x26) + { + Put16(page, 0x30 + field, (ushort)value); + } + else + { + Put32(page, 0x30 + field, (uint)value); + } + }); + AssertReplayRefused(volume, "implausible restart area"); + } + + [Fact] + public void ALogSequenceThatWouldWrapPastItsMaximum_IsRefused() + { + using var volume = Volume(); + var geometry = new TestJournalWriter(volume); + var writer = new TestJournalWriter(volume, startOffset: geometry.FileSize - TestJournalWriter.PageSize, + sequence: (1UL << geometry.SequenceNumberBits) - 1); + var restart = writer.Checkpoint([]); + writer.AppendRaw(new byte[TestJournalWriter.PageSize], LfsRecord.TypeClientRestart, 0, 0); + writer.Finish(restart); + AssertReplayRefused(volume, "overflows the log sequence"); + } + + [Theory] + [InlineData(false, 0)] + [InlineData(false, 1)] + [InlineData(false, 2)] + [InlineData(false, 3)] + [InlineData(true, 0)] + [InlineData(true, 1)] + public void ReplayFlushesMetadataBeforeTheResetAndPropagatesFlushFailures(bool alreadyApplied, int failFlush) + { + using var probe = Volume(); + var target = FreeCluster(probe); + using var dirty = WithRecords(Raw(target, Pattern), Forget()); + if (alreadyApplied) + { + dirty.Position = target * 4096; + dirty.Write(Pattern, 0, Pattern.Length); + } + + var initial = dirty.ToArray(); + using var volume = new ReplayTrackingStream((byte[])initial.Clone(), failFlush); + if (failFlush == 0) + { + var result = NtfsJournal.Replay(volume); + Assert.Equal(alreadyApplied ? NtfsJournalStatus.NothingToReplay : NtfsJournalStatus.NeedsReplay, result.Status); + Assert.True(result.Replayed); + } + else + { + Assert.Same(volume.Failure, Assert.Throws(() => NtfsJournal.Replay(volume))); + } + + var expected = new List(); + if (!alreadyApplied) + { + expected.Add($"write {target * 4096}"); + } + + expected.Add("flush"); + if (failFlush == 1) + { + Assert.Equal(expected, volume.Events); + var original = new TestJournalWriter(dirty).Storage; + var result = new TestJournalWriter(volume).Storage; + var first = new byte[2 * TestJournalWriter.PageSize]; + var second = new byte[first.Length]; + original.Read(0, first, 0, first.Length); + result.Read(0, second, 0, second.Length); + Assert.Equal(first, second); + return; + } + + Assert.Equal(expected, volume.Events.Take(expected.Count)); + var resetEvents = volume.Events.Skip(expected.Count).ToArray(); + Assert.Equal(failFlush == 2 ? 2 : 4, resetEvents.Length); + for (var i = 0; i < resetEvents.Length; i += 2) + { + Assert.StartsWith("write ", resetEvents[i]); + Assert.Equal("flush", resetEvents[i + 1]); + } + } + + [Fact] + public void FileStreamReplay_RequestsDiskFlushesAtEveryBoundary() + { + using var dirty = WithRecords(Forget()); + var path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, dirty.ToArray()); + using var file = new ReplayTrackingFile(path); + Assert.True(NtfsJournal.Replay(file).Replayed); + Assert.Equal(new[] { true, true, true }, file.DiskFlushes); + } + finally + { + File.Delete(path); + } + } + + [Theory] + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(1, true)] + public void PartitionReplayForwardsDiskFlushThroughSparseWrappers(int ranges, bool derivedFile) + { + using var dirty = WithRecords(Forget()); + var path = Path.GetTempFileName(); + try + { + const int padding = 512; + using (var destination = File.OpenWrite(path)) + { + destination.Write(new byte[padding * ranges], 0, padding * ranges); + dirty.WriteTo(destination); + } + + using var file = derivedFile ? new ReplayTrackingFile(path) : new FileStream(path, FileMode.Open, FileAccess.ReadWrite); + using var partition = new SubStream(file, padding, file.Length - padding); + using var nested = ranges == 2 ? new SubStream(partition, padding, dirty.Length) : null; + Stream volume = nested ?? partition; + Assert.True(NtfsJournal.Replay(volume).Replayed); + if (file is ReplayTrackingFile tracking) + { + Assert.Equal(new[] { true, true, true }, tracking.DiskFlushes); + Assert.Equal(0, tracking.StreamFlushes); + } + + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void ReplayUsesAnOverriddenSparseStreamDiskFlush() + { + using var dirty = WithRecords(Forget()); + using var partition = new TrackingSubStream(dirty); + using var volume = new SubStream(partition, 0, partition.Length); + Assert.True(NtfsJournal.Replay(volume).Replayed); + Assert.Equal(3, partition.Flushes); + Assert.Equal(new[] { true, true, true }, partition.DiskFlushes); + } + + [Fact] + public void SparseStreamDiskFlushFallsBackForAnUnknownBackingStream() + { + using var backing = new ReplayTrackingStream(new byte[512], failFlush: int.MaxValue); + using var wrapped = SparseStream.FromStream(backing, Ownership.None); + wrapped.Flush(flushToDisk: true); + Assert.Equal(new[] { "flush" }, backing.Events); + } + + [Fact] + public void SparseStreamDiskFlushPropagatesThroughSynchronizationAndCacheWrappers() + { + using var backing = new MemoryStream(new byte[512], writable: true); + using var tracked = new TrackingSubStream(backing); + using var wrapped = new WrappingStream(tracked, Ownership.None); + using var synchronized = SparseStream.Synchronized(wrapped, Ownership.None); + using var threadSafe = new ThreadSafeStream(synchronized, Ownership.None); + using var cache = new BlockCacheStream(threadSafe, Ownership.None); + + cache.Flush(flushToDisk: true); + Assert.Equal(new[] { true }, tracked.DiskFlushes); + } + + [Fact] + public void DynamicVhdReplayPropagatesFlushAndStopsBeforePublishingAResetOnFailure() + { + using var dirty = Volume(); + var (writer, _, _, _) = WriteRedoInterruptionFixture(dirty); + using var backing = new MemoryStream(); + using (var disk = DiscUtils.Vhd.Disk.InitializeDynamic(backing, Ownership.None, dirty.Length)) + { + dirty.WriteTo(disk.Content); + } + + using var tracked = new ReplayTrackingStream(backing.ToArray(), failFlush: 2); + using var reopened = new DiscUtils.Vhd.Disk(tracked, Ownership.None); + using var volume = new SubStream(reopened.Content, 0, dirty.Length); + var originalRestart = new byte[2 * TestJournalWriter.PageSize]; + writer.Storage.Read(0, originalRestart, 0, originalRestart.Length); + + Assert.Same(tracked.Failure, Assert.Throws(() => NtfsJournal.Replay(volume))); + Assert.Equal(2, tracked.Events.Count(e => e == "flush")); + var actualRestart = new byte[originalRestart.Length]; + LogFileStorage.Open(volume, writer.Layout).Read(0, actualRestart, 0, actualRestart.Length); + Assert.Equal(originalRestart, actualRestart); + + Assert.True(NtfsJournal.Replay(volume).Replayed); + Assert.Equal(5, tracked.Events.Count(e => e == "flush")); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + } + + [Fact] + public void DynamicVhdReplayRequestsDiskFlushFromFileBacking() + { + using var dirty = WithRecords(Forget()); + var path = Path.GetTempFileName(); + try + { + using (var destination = new FileStream(path, FileMode.Create, FileAccess.ReadWrite)) + using (var disk = DiscUtils.Vhd.Disk.InitializeDynamic(destination, Ownership.None, dirty.Length)) + { + dirty.WriteTo(disk.Content); + } + + using var file = new ReplayTrackingFile(path); + using var reopened = new DiscUtils.Vhd.Disk(file, Ownership.None); + Assert.True(NtfsJournal.Replay(reopened.Content).Replayed); + Assert.Equal(new[] { true, true, true }, file.DiskFlushes); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(reopened.Content).Status); + } + finally + { + File.Delete(path); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AFailedMirrorWriteOrFlush_LeavesThePrimaryRecordAvailableForRetry(bool failFlush) + { + using var dirty = Volume(); + var layout = WriteFirstMftRecordUpdate(dirty); + var mft = layout.MftCluster * layout.BytesPerCluster; + var mirror = layout.MftMirrorCluster * layout.BytesPerCluster; + Assert.True(mft < mirror); + var original = Read(dirty, mft, layout.MftRecordSize); + + using var interrupted = new ReplayTrackingStream(dirty.ToArray(), failFlush ? 1 : 0, failFlush ? null : mirror); + Assert.Same(interrupted.Failure, Assert.Throws(() => NtfsJournal.Replay(interrupted))); + Assert.Equal($"write {mirror}", interrupted.Events[0]); + Assert.Equal(original, Read(interrupted, mft, layout.MftRecordSize)); + + using var retry = JournalScenario.Copy(interrupted.ToArray()); + Assert.Equal(NtfsJournalStatus.NeedsReplay, NtfsJournal.Replay(retry).Status); + Assert.Equal(Read(retry, mft, layout.MftRecordSize), Read(retry, mirror, layout.MftRecordSize)); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(retry).Status); + } + + [Fact] + public void LargeClusterReplayFlushesTheMirrorOfMftRecordFourBeforeWritingThePrimary() + { + // Use the Windows fixture's geometry and mirror layout, with a synthetic journal update to $AttrDef. + using var dirty = NtfsJournalWindowsSampleTest.LoadVolume("s64k"); + var writer = new TestJournalWriter(dirty, sequence: 1000); + var layout = writer.Layout; + var recordOffset = 4 * layout.MftRecordSize; + var mft = layout.MftCluster * layout.BytesPerCluster; + var mirror = layout.MftMirrorCluster * layout.BytesPerCluster; + var original = Read(dirty, mft + recordOffset, layout.MftRecordSize); + var reservedBytes = 16 * layout.MftRecordSize; + var mirrorTail = Read(dirty, mirror + reservedBytes, layout.BytesPerCluster - reservedBytes); + Assert.False(mirrorTail.AsSpan().SequenceEqual(Read(dirty, mft + reservedBytes, mirrorTail.Length))); + var change = FirstMftValueUpdate(writer, original, Pattern.AsSpan(0, 8).ToArray(), new byte[8]); + change.ClusterBlockOffset = recordOffset / 512; + var restart = writer.Checkpoint(Attributes); + var lsn = writer.Append(change); + writer.Append(Forget()); + writer.Finish(restart); + + using var volume = new ReplayTrackingStream(dirty.ToArray(), failFlush: 1); + Assert.Same(volume.Failure, Assert.Throws(() => NtfsJournal.Replay(volume))); + Assert.Equal(new[] { $"write {mirror}", "flush" }, volume.Events); + Assert.Equal(original, Read(volume, mft + recordOffset, layout.MftRecordSize)); + Assert.Equal(lsn, U64(Read(volume, mirror + recordOffset, layout.MftRecordSize), 8)); + + Assert.True(NtfsJournal.Replay(volume).Replayed); + Assert.Equal(Read(volume, mft + recordOffset, layout.MftRecordSize), + Read(volume, mirror + recordOffset, layout.MftRecordSize)); + Assert.Equal(mirrorTail, Read(volume, mirror + reservedBytes, mirrorTail.Length)); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + } + + private sealed class ReplayTrackingStream(byte[] image, int failFlush, long? failWrite = null) : MemoryStream(image, writable: true) + { + private int _flushes; + + public IOException Failure { get; } = new("Injected replay I/O failure"); + + public List Events { get; } = []; + + public override void Write(byte[] buffer, int offset, int count) + { + Events.Add($"write {Position}"); + if (Position == failWrite) + { + throw Failure; + } + + base.Write(buffer, offset, count); + } + + public override void Flush() + { + Events.Add("flush"); + if (++_flushes == failFlush) + { + throw Failure; + } + } + } + + private sealed class ReplayTrackingFile(string path) : FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None) + { + public List DiskFlushes { get; } = []; + + public int StreamFlushes { get; private set; } + + public override void Flush() + { + StreamFlushes++; + base.Flush(); + } + + public override void Flush(bool flushToDisk) + { + DiskFlushes.Add(flushToDisk); + base.Flush(flushToDisk); + } + } + + private sealed class TrackingSubStream(Stream parent) : SubStream(parent, 0, parent.Length) + { + public int Flushes { get; private set; } + + public List DiskFlushes { get; } = []; + + public override void Flush() + { + Flushes++; + base.Flush(); + } + + public override void Flush(bool flushToDisk) + { + DiskFlushes.Add(flushToDisk); + Flush(); + } + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalScenarioTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalScenarioTest.cs new file mode 100644 index 000000000..51e6ed408 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalScenarioTest.cs @@ -0,0 +1,332 @@ +using System; +using System.IO; +using System.Linq; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Replays journals written by for changes DiscUtils made itself: the replay must +/// turn the "before" volume into the "after" one (redo) and the other way round (undo). +/// +public class NtfsJournalScenarioTest +{ + private static void AssertStatus(NtfsJournalStatus expected, NtfsJournalInfo info) => + Assert.True(info.Status == expected, $"expected {expected}, got {info}"); + + private static void CreateFiles(NtfsFileSystem ntfs) + { + ntfs.CreateDirectory(@"dir\sub"); + for (var i = 0; i < 40; i++) + { + using var s = ntfs.OpenFile($@"dir\file{i:D3}.txt", FileMode.Create); + var content = Enumerable.Repeat((byte)('a' + (i % 26)), 100 + (i * 97)).ToArray(); + s.Write(content, 0, content.Length); + } + + using (var big = ntfs.OpenFile(@"dir\sub\big.bin", FileMode.Create)) + { + big.Write(new byte[20000], 0, 20000); + } + + ntfs.SetAttributes(@"dir\file001.txt", FileAttributes.Hidden); + } + + private static void ChangeFiles(NtfsFileSystem ntfs) + { + ntfs.DeleteFile(@"dir\file002.txt"); + ntfs.MoveFile(@"dir\file003.txt", @"dir\sub\moved.txt"); + using (var s = ntfs.OpenFile(@"dir\file004.txt", FileMode.Append)) + { + s.Write(new byte[5000], 0, 5000); + } + + ntfs.CreateDirectory(@"dir\new"); + ntfs.DeleteFile(@"dir\sub\big.bin"); + } + + [Fact] + public void Redo_ACommittedTransaction_BringsTheVolumeUpToDate() + { + var scenario = JournalScenario.Create(CreateFiles); + Assert.NotEmpty(scenario.Records); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + + var analysis = NtfsJournal.Analyze(volume); + AssertStatus(NtfsJournalStatus.NeedsReplay, analysis); + Assert.True(scenario.Records.Count == analysis.RecordsToRedo, $"{scenario.Records.Count} records, {analysis}"); + Assert.Equal(0, analysis.TransactionsToUndo); + + var replay = NtfsJournal.Replay(volume); + Assert.True(replay.Replayed); + scenario.AssertVolumeEquals(scenario.After, volume); + AssertStatus(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume)); + + using var ntfs = new NtfsFileSystem(volume); + Assert.Equal(39, ntfs.GetFiles(@"dir").Count()); + Assert.Equal(20000, ntfs.GetFileLength(@"dir\sub\big.bin")); + } + + [Fact] + public void Undo_AnUnfinishedTransaction_RollsTheVolumeBack() + { + var scenario = JournalScenario.Create(ChangeFiles, setup: CreateFiles); + using var volume = JournalScenario.Copy(scenario.After); + scenario.WriteJournal(volume, committed: false); + + var analysis = NtfsJournal.Analyze(volume); + AssertStatus(NtfsJournalStatus.NeedsReplay, analysis); + Assert.Equal(1, analysis.TransactionsToUndo); + Assert.Equal(scenario.Records.Count, analysis.RecordsToUndo); + Assert.False(analysis.CanReplayPersistently); + + var before = volume.ToArray(); + Assert.Contains("metadata undo", Assert.Throws(() => NtfsJournal.Replay(volume)).Message); + Assert.Equal(before, volume.ToArray()); + + NtfsJournal.ReplayInMemory(volume, out var overlay); + using (overlay) + { + scenario.AssertVolumeEquals(scenario.Before, overlay); + } + + using var ntfs = new NtfsFileSystem(new MemoryStream(before, writable: false)); + Assert.True(ntfs.JournalInfo.ReplayedInMemory); + Assert.True(ntfs.FileExists(@"dir\file002.txt")); + Assert.True(ntfs.FileExists(@"dir\sub\big.bin")); + Assert.False(ntfs.DirectoryExists(@"dir\new")); + } + + [Fact] + public void Redo_RecordsThatWrapAroundTheEndOfTheLog() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + var size = new TestJournalWriter(volume).FileSize; + var writer = scenario.WriteJournal(volume, committed: true, startOffset: size - (3 * TestJournalWriter.PageSize)); + Assert.True(writer.SequenceNumberBits > 0); + Assert.True(scenario.Records.Count * 1024 > 3 * TestJournalWriter.PageSize, "the journal must be longer than the space left before the end"); + + NtfsJournal.Replay(volume); + scenario.AssertVolumeEquals(scenario.After, volume); + } + + [Fact] + public void Redo_LogFileServiceVersion1() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true, majorVersion: 1); + + Assert.Equal("1.1", NtfsJournal.Analyze(volume).LogVersion); + NtfsJournal.Replay(volume); + scenario.AssertVolumeEquals(scenario.After, volume); + } + + [Fact] + public void Replay_IsIdempotent_AndAnInterruptedReplayReplaysToTheSameResult() + { + var scenario = JournalScenario.Create(ChangeFiles, setup: CreateFiles); + using var crashed = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(crashed, committed: true); + var crashedImage = crashed.ToArray(); + + // An interruption after the pages were written but before the journal was reset: the pages are replayed, + // the journal is still the crash journal. + using var interrupted = JournalScenario.Copy(crashedImage); + NtfsJournal.Replay(interrupted); + var replayed = interrupted.ToArray(); + foreach (var lcn in scenario.LogClusters) + { + Array.Copy(crashedImage, lcn * scenario.ClusterSize, replayed, lcn * scenario.ClusterSize, scenario.ClusterSize); + } + + using var again = JournalScenario.Copy(replayed); + AssertStatus(NtfsJournalStatus.NothingToReplay, NtfsJournal.Analyze(again)); + NtfsJournal.Replay(again); + scenario.AssertVolumeEquals(scenario.After, again); + + // And a replay of a replayed volume does nothing. + var info = NtfsJournal.Replay(again); + AssertStatus(NtfsJournalStatus.Clean, info); + Assert.False(info.Replayed); + } + + [Fact] + public void ReadOnlyOpen_ShowsTheReplayedVolume_WithoutWritingIt() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + var image = volume.ToArray(); + + using var readOnly = new MemoryStream(image, writable: false); + using (var ntfs = new NtfsFileSystem(readOnly)) + { + AssertStatus(NtfsJournalStatus.NeedsReplay, ntfs.JournalInfo); + Assert.False(ntfs.JournalInfo.Replayed); + Assert.True(ntfs.JournalInfo.ReplayedInMemory); + Assert.True(ntfs.FileExists(@"dir\file039.txt")); + Assert.Equal(100 + (39 * 97), ntfs.GetFileLength(@"dir\file039.txt")); + } + + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void ExplicitWritableReplay_ReplaysAndResetsTheJournal() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + + using (var ntfs = new NtfsFileSystem(volume, NtfsJournalMode.Replay)) + { + Assert.True(ntfs.JournalInfo.Replayed); + Assert.True(ntfs.FileExists(@"dir\sub\big.bin")); + } + + scenario.AssertVolumeEquals(scenario.After, volume); + AssertStatus(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume)); + } + + [Fact] + public void RefuseMode_ThrowsAndWritesNothing() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + var image = volume.ToArray(); + + Assert.Throws(() => new NtfsFileSystem(volume, NtfsJournalMode.Refuse)); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void IgnoreMode_OpensTheVolumeAsItIs() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true); + var image = volume.ToArray(); + + using (var ntfs = new NtfsFileSystem(volume, NtfsJournalMode.Ignore)) + { + Assert.Null(ntfs.JournalInfo); + Assert.False(ntfs.DirectoryExists(@"dir")); + } + + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void ACommitThatNeverReachedTheDisk_IsRolledBack() + { + // The pages of the transaction are on disk, but the page holding its last records (and the commit) is torn. + var scenario = JournalScenario.Create(ChangeFiles, setup: CreateFiles); + using var volume = JournalScenario.Copy(scenario.After); + var writer = scenario.WriteJournal(volume, committed: true); + var lastPage = writer.LsnToOffsetForTest(writer.LastLsn) & ~(long)(TestJournalWriter.PageSize - 1); + var page = new byte[TestJournalWriter.PageSize]; + writer.Storage.Read(lastPage, page, 0, page.Length); + page[0x1FE] ^= 0xFF; + writer.Storage.Write(lastPage, page, 0, page.Length); + + var analysis = NtfsJournal.Analyze(volume); + Assert.Equal(1, analysis.TransactionsToUndo); + Assert.False(analysis.CanReplayPersistently); + + using var readOnly = new MemoryStream(volume.ToArray(), writable: false); + using var ntfs = new NtfsFileSystem(readOnly); + Assert.True(ntfs.FileExists(@"dir\file002.txt")); + Assert.False(ntfs.DirectoryExists(@"dir\new")); + } + + [Fact] + public void AnOperationDiscUtilsDoesNotKnow_IsRefused_AndNothingIsWritten() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.Before); + scenario.WriteJournal(volume, committed: true, adjust: r => r.TargetAttribute == JournalScenario.MftData + ? new TestLogRecord + { + Redo = NtfsLogOperation.UpdateRelativeDataInIndex, + Undo = NtfsLogOperation.Noop, + TransactionId = r.TransactionId, + RedoData = new byte[16], + TargetAttribute = r.TargetAttribute, + TargetVcn = r.TargetVcn, + ClusterBlockOffset = r.ClusterBlockOffset, + Lcns = r.Lcns, + } + : r); + var image = volume.ToArray(); + + var analysis = NtfsJournal.Analyze(volume); + AssertStatus(NtfsJournalStatus.Unsupported, analysis); + Assert.Contains("UpdateRelativeDataInIndex", analysis.Detail); + Assert.Throws(() => NtfsJournal.Replay(volume)); + Assert.Throws(() => new NtfsFileSystem(volume)); + Assert.True(image.AsSpan().SequenceEqual(volume.ToArray())); + } + + [Fact] + public void Clean_EmptyAndChkdskJournals_NeedNothing() + { + var scenario = JournalScenario.Create(CreateFiles); + + // As formatted: $LogFile is all 0xFF. + using (var volume = JournalScenario.Copy(scenario.After)) + { + var info = NtfsJournal.Analyze(volume); + AssertStatus(NtfsJournalStatus.Clean, info); + Assert.Contains("empty", info.Detail); + } + + // Shut down cleanly. + using (var volume = JournalScenario.Copy(scenario.After)) + { + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint([]); + writer.Finish(restart, clean: true); + AssertStatus(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume)); + } + + // chkdsk obsoleted the journal. + using (var volume = JournalScenario.Copy(scenario.Before)) + { + scenario.WriteJournal(volume, committed: true); + var storage = new TestJournalWriter(volume).Storage; + var page = new byte[TestJournalWriter.PageSize]; + storage.Read(0, page, 0, page.Length); + EndianUtilities.WriteBytesLittleEndian(JournalUtilities.SignatureChkd, page, 0); + storage.Write(0, page, 0, page.Length); + var info = NtfsJournal.Analyze(volume); + AssertStatus(NtfsJournalStatus.Clean, info); + Assert.Contains("chkdsk", info.Detail); + } + } + + [Fact] + public void NotShutDownCleanly_ButNothingPending_IsMarkedClean() + { + var scenario = JournalScenario.Create(CreateFiles); + using var volume = JournalScenario.Copy(scenario.After); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint([(JournalScenario.MftData, 0x80, 0x0001000000000000UL, 0, "")]); + writer.Append(new TestLogRecord { Redo = NtfsLogOperation.ForgetTransaction, Undo = NtfsLogOperation.CompensationLogRecord, TransactionId = 0x40 }); + writer.Finish(restart); + + var info = NtfsJournal.Analyze(volume); + AssertStatus(NtfsJournalStatus.NothingToReplay, info); + + var image = volume.ToArray(); + info = NtfsJournal.Replay(volume); + Assert.True(info.Replayed); + scenario.AssertVolumeEquals(image, volume); + AssertStatus(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume)); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalUndoTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalUndoTest.cs new file mode 100644 index 000000000..1be2c2de2 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalUndoTest.cs @@ -0,0 +1,317 @@ +using System; +using System.IO; +using System.Linq; +using DiscUtils.Ntfs; +using DiscUtils.Ntfs.Journal; +using static DiscUtils.Ntfs.Journal.JournalUtilities; + +namespace LibraryTests.Ntfs.Journal; + +public partial class NtfsJournalEdgeCaseTest +{ + [Fact] + public void UndoAppliesEveryInverseOnTheSameFileRecord() + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var position = writer.Layout.MftCluster * writer.Layout.BytesPerCluster; + var original = Read(volume, position, writer.Layout.MftRecordSize, logical: true); + var attribute = U16(original, 0x14); + var originalValue = original.AsSpan(attribute + 0x18, 8).ToArray(); + var middle = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + var final = new byte[] { 8, 7, 6, 5, 4, 3, 2, 1 }; + var restart = writer.Checkpoint(Attributes); + var first = writer.Append(FirstMftValueUpdate(writer, original, middle, originalValue)); + var last = FirstMftValueUpdate(writer, original, final, middle); + last.UndoNextLsn = first; + var lastLsn = writer.Append(last); + writer.Finish(restart); + var before = volume.ToArray(); + + for (var attempt = 0; attempt < 2; attempt++) + { + var info = NtfsJournal.ReplayInMemory(volume, out var overlay); + using var recovered = overlay; + Assert.Equal(NtfsJournalStatus.NeedsReplay, info.Status); + Assert.False(info.CanReplayPersistently); + Assert.True(info.ReplayedInMemory); + Assert.Equal(2, info.RecordsToUndo); + var restored = Read(recovered, position, original.Length, logical: true); + Assert.Equal(originalValue, restored.AsSpan(attribute + 0x18, 8).ToArray()); + Assert.Equal(lastLsn, U64(restored, 8)); + } + + Assert.Contains("metadata undo", Assert.Throws(() => NtfsJournal.Replay(volume)).Message); + Assert.Equal(before, volume.ToArray()); + } + + private static TestLogRecord FirstMftValueUpdate(TestJournalWriter writer, byte[] record, byte[] redo, byte[] undo) => + Op(NtfsLogOperation.UpdateResidentValue, redo, Mft, 0, [writer.Layout.MftCluster], + recordOffset: U16(record, 0x14), attributeOffset: 0x18, undo: NtfsLogOperation.UpdateResidentValue, undoData: undo); + + [Fact] + public void PendingUndoCannotResetTheJournalEvenWhenTheNetChangeIsZero() + { + using var volume = Volume(); + using var dirty = WithRecordsOn(volume, Raw(FreeCluster(volume), Pattern)); + var before = dirty.ToArray(); + var info = NtfsJournal.Analyze(dirty); + Assert.Equal(NtfsJournalStatus.NothingToReplay, info.Status); + Assert.Equal(1, info.TransactionsToUndo); + Assert.False(info.CanReplayPersistently); + Assert.Contains("metadata undo", Assert.Throws(() => NtfsJournal.Replay(dirty)).Message); + Assert.Equal(before, dirty.ToArray()); + } + + [Theory] + [InlineData(1UL)] + [InlineData(0x100UL)] + public void APageLsnBeyondTheLogCannotStandInForUndoProgress(ulong beyondEnd) + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var position = writer.Layout.MftCluster * writer.Layout.BytesPerCluster; + var record = Read(volume, position, writer.Layout.MftRecordSize, logical: true); + var attribute = U16(record, 0x14); + var old = record.AsSpan(attribute + 0x18, 8).ToArray(); + var restart = writer.Checkpoint(Attributes); + writer.Append(FirstMftValueUpdate(writer, record, Pattern.AsSpan(0, 8).ToArray(), old)); + writer.Finish(restart); + Put64(record, 8, writer.LastLsn + beyondEnd); + ApplyFixups(record, 0, record.Length); + volume.Position = position; + volume.Write(record, 0, record.Length); + + AssertReplayRefused(volume, "cannot establish the undo history"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ActiveTransactionsCanBeUndoneOnlyOnDisjointClusters(bool overlaps) + { + using var volume = Volume(); + var x = FreeCluster(volume); + var y = overlaps ? x : FreeCluster(volume, 1); + var after = Enumerable.Repeat((byte)0x75, Pattern.Length).ToArray(); + using var dirty = WithRecordsOn(volume, Raw(x, Pattern), Raw(y, after, tx: 0x68)); + if (overlaps) + { + AssertReplayRefused(dirty, "overlaps cluster"); + } + else + { + var info = NtfsJournal.Analyze(dirty); + Assert.Equal(NtfsJournalStatus.NothingToReplay, info.Status); + Assert.Equal(2, info.TransactionsToUndo); + Assert.Equal(2, info.RecordsToUndo); + Assert.False(info.CanReplayPersistently); + } + } + + [Theory] + [InlineData(4, 4, false, null)] + [InlineData(4, 4, true, null)] + [InlineData(30000, 9, true, null)] + [InlineData(2, 4, true, "overlaps bits 2-3 of cluster")] + [InlineData(-1, 0, true, "overlaps cluster")] + public void ActiveTransactionsCanRollBackDifferentBitsOfOneBitmapCluster(int first, int count, bool onDisk, string? refused) + { + // Two unfinished allocations in the same bitmap cluster, as in a real crash with two active transactions + // that each allocated a cluster in the same 128 MB of a volume. Their bits either never reached the disk (as + // in that crash: redo sets them and undo clears them again) or did. The second transaction sets other bits, + // some of the same bits, or rewrites the whole cluster. + using var volume = Volume(); + var lcn = FreeCluster(volume); + if (onDisk) + { + SetBitsOnDisk(volume, lcn, 0, 4); + if (first >= 0) + { + SetBitsOnDisk(volume, lcn, first, count); + } + } + + var other = first >= 0 ? AllocationBits(lcn, first, count, 0x68) : Raw(lcn, Pattern, tx: 0x68); + using var dirty = WithRecordsOn(volume, AllocationBits(lcn, 0, 4, Tx), other); + if (refused != null) + { + AssertReplayRefused(dirty, refused); + return; + } + + var before = dirty.ToArray(); + for (var attempt = 0; attempt < 2; attempt++) + { + var info = NtfsJournal.ReplayInMemory(dirty, out var overlay); + using var recovered = overlay; + Assert.Equal(onDisk ? NtfsJournalStatus.NeedsReplay : NtfsJournalStatus.NothingToReplay, info.Status); + Assert.Equal(2, info.TransactionsToUndo); + Assert.Equal(2, info.RecordsToUndo); + Assert.False(info.CanReplayPersistently); + Assert.Equal(onDisk, info.ReplayedInMemory); + Assert.Equal(new byte[4096], Read(recovered ?? (Stream)dirty, lcn * 4096, 4096)); + } + + Assert.Contains("metadata undo", Assert.Throws(() => NtfsJournal.Replay(dirty)).Message); + Assert.Equal(before, dirty.ToArray()); + } + + private static void SetBitsOnDisk(Stream volume, long lcn, int first, int count) + { + for (var bit = first; bit < first + count; bit++) + { + var position = (lcn * 4096) + (bit / 8); + volume.Position = position; + var value = volume.ReadByte(); + volume.Position = position; + volume.WriteByte((byte)(value | (1 << (bit % 8)))); + } + } + + private static TestLogRecord AllocationBits(long lcn, int first, int count, uint tx) + { + var run = BitConverter.GetBytes((uint)first | ((long)count << 32)); + var record = Op(NtfsLogOperation.SetBitsInNonresidentBitMap, run, Data, lcn, [lcn], + undo: NtfsLogOperation.ClearBitsInNonresidentBitMap, undoData: run); + record.TransactionId = tx; + return record; + } + + [Fact] + public void ARecordedCompensationCanBeRedoneWithoutInventingUndoProgress() + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var target = FreeCluster(volume); + var restart = writer.Checkpoint(Attributes); + writer.Append(Raw(target, Pattern)); + writer.Append(Op(NtfsLogOperation.UpdateNonresidentValue, new byte[Pattern.Length], Data, target, [target], + undo: NtfsLogOperation.CompensationLogRecord)); + writer.Finish(restart); + volume.Position = target * 4096; + volume.Write(Pattern, 0, Pattern.Length); + + var info = NtfsJournal.Replay(volume); + Assert.Equal(0, info.TransactionsToUndo); + Assert.True(info.CanReplayPersistently); + Assert.True(info.Replayed); + Assert.Equal(new byte[Pattern.Length], Read(volume, target * 4096, Pattern.Length)); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + } + + [Fact] + public void ACompensationWithoutAMetadataMappingCannotDiscardUndo() + { + using var volume = Volume(); + var target = FreeCluster(volume); + using var dirty = WithRecordsOn(volume, Raw(target, Pattern), + Op(NtfsLogOperation.UpdateNonresidentValue, new byte[Pattern.Length], Data, target, [], + undo: NtfsLogOperation.CompensationLogRecord)); + AssertReplayRefused(dirty, "changes metadata without a cluster mapping"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void APreparedTransactionWithoutAnOutcomeIsRefused(bool inCheckpoint) + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var record = Raw(FreeCluster(volume), Pattern); + ulong restart; + if (inCheckpoint) + { + var lsn = writer.Append(record); + restart = writer.Checkpoint(Attributes, transactions: [(Tx, TransactionState.Prepared, lsn, lsn)]); + } + else + { + restart = writer.Checkpoint(Attributes); + writer.Append(record); + writer.Append(new TestLogRecord { Redo = NtfsLogOperation.PrepareTransaction, TransactionId = Tx }); + } + + writer.Finish(restart); + AssertReplayRefused(volume, "has no recorded outcome"); + } + + [Theory] + [InlineData(0)] + [InlineData(255)] + public void AnAllocatedCheckpointEntryMustHaveARecognizedState(int state) + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var lsn = writer.Append(Raw(FreeCluster(volume), Pattern)); + var restart = writer.Checkpoint(Attributes, transactions: [(Tx, (TransactionState)state, lsn, lsn)]); + writer.Finish(restart); + AssertReplayRefused(volume, "invalid checkpoint state"); + } + + [Theory] + [InlineData(LfsRecord.TypeClientRecord, 0, 0, "does not belong to the active NTFS client")] + [InlineData(LfsRecord.TypeClientRecord, 1, ActiveClientSequence, "does not belong to the active NTFS client")] + [InlineData(LfsRecord.TypeClientRestart, 0, ActiveClientSequence, "wrong LFS record type")] + public void UndoPointersMustNameRecordsOfTheActiveClient(uint recordType, int clientIndex, int clientSequence, string reason) + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume) { ClientSequenceNumber = ActiveClientSequence }; + var restart = writer.Checkpoint(Attributes); + var record = Raw(FreeCluster(volume), Pattern); + writer.Append(record); + record.UndoNextLsn = writer.AppendRaw(record.ClientData(), recordType, Tx, 0, + clientSequence: (ushort)clientSequence, clientIndex: (ushort)clientIndex); + writer.Append(record); + writer.Finish(restart); + if (clientIndex != 0) + { + AddFreeClientSlot(writer); + } + + AssertReplayRefused(volume, reason); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AnUndoChainCannotGoBackToAnEarlierUseOfItsTransactionSlot(bool missing) + { + using var volume = Volume(); + var writer = new TestJournalWriter(volume); + var target = FreeCluster(volume); + var restart = writer.Checkpoint(Attributes); + var prior = writer.Append(Raw(target, Pattern)); + writer.Append(Forget()); + var current = Raw(target, Pattern); + current.UndoNextLsn = missing ? writer.LsnOf(writer.FirstPageOffset - 8, 1) : prior; + writer.Append(current); + writer.Finish(restart); + AssertReplayRefused(volume, missing ? "not in the log" : "outside the undo history"); + } + + [Fact] + public void MetadataUndoRequiresAKnownAndUnchangedClusterMapping() + { + using (var volume = Volume()) + { + var missing = Raw(FreeCluster(volume), Pattern); + missing.Lcns = []; + using var dirty = WithRecordsOn(volume, missing); + AssertReplayRefused(dirty, "metadata undo without a cluster mapping"); + } + + using (var volume = Volume()) + { + var target = FreeCluster(volume); + var writer = new TestJournalWriter(volume); + var restart = writer.Checkpoint(Attributes); + var change = writer.Append(Raw(target, Pattern)); + var hotFix = HotFix(target, [FreeCluster(volume, 1)]); + hotFix.UndoNextLsn = change; + writer.Append(hotFix); + writer.Finish(restart); + AssertReplayRefused(volume, "remapped cluster in its undo target"); + } + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsJournalWindowsSampleTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalWindowsSampleTest.cs new file mode 100644 index 000000000..2f12572fd --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsJournalWindowsSampleTest.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using DiscUtils.Ntfs; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Volumes Windows really crashed on, checked in as test data, each with a digest of Windows' own replay of the +/// same crash (see ). +/// +/// +/// Made in a Hyper-V VM (Windows 11 Enterprise LTSC 2024, build 26100): a small data disk formatted NTFS in the +/// guest, several jobs creating, writing, renaming and deleting files, directories, hard links and alternate data +/// streams, and the VM powered off at a random moment. Windows' replay is a copy of the crashed disk attached to the +/// same VM, brought online (which replays the journal) and taken offline again without touching any file. +/// +public class NtfsJournalWindowsSampleTest +{ + /// Sample, what the journal needs, and the cluster size. + /// + /// s512: 512-byte clusters (file records and index buffers span clusters). s64k: 64K clusters (several dirty + /// pages per cluster). x4k-a, x4k-c: compressed and sparse files, object ids, symbolic links and quota tracking + /// (Windows' zero-data records). s4kn: a disk with 4096-byte sectors, flushed before the crash. + /// + public static IEnumerable Samples => + [ + ["s512", NtfsJournalStatus.NeedsReplay, 512], + ["s64k", NtfsJournalStatus.NeedsReplay, 65536], + ["x4k-a", NtfsJournalStatus.NeedsReplay, 4096], + ["x4k-c", NtfsJournalStatus.NeedsReplay, 4096], + ["s4kn", NtfsJournalStatus.NothingToReplay, 4096], + ]; + + internal static MemoryStream LoadVolume(string name) => Load($"journal-{name}.ntfs.gz"); + + internal static List LoadDigest(string name) + { + using var stream = Load($"journal-{name}.windows.txt.gz"); + using var reader = new StreamReader(stream); + var lines = new List(); + while (reader.ReadLine() is { } line) + { + lines.Add(line); + } + + return lines; + } + + private static MemoryStream Load(string file) + { + // The data lives next to the sources; look for it upwards from wherever the tests run. + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory != null) + { + foreach (var candidate in new[] + { + Path.Combine(directory.FullName, "LibraryTests", "Ntfs", "Data", file), + Path.Combine(directory.FullName, "Tests", "LibraryTests", "Ntfs", "Data", file), + }) + { + if (System.IO.File.Exists(candidate)) + { + using var gz = new GZipStream(System.IO.File.OpenRead(candidate), CompressionMode.Decompress); + var volume = new MemoryStream(); + gz.CopyTo(volume); + volume.Position = 0; + return volume; + } + } + + directory = directory.Parent; + } + + throw new FileNotFoundException($"Test data {file} not found above {AppContext.BaseDirectory}"); + } + + [Theory] + [MemberData(nameof(Samples))] + public void Analyze_ClassifiesTheCrashLikeWindows(string name, NtfsJournalStatus expected, int clusterSize) + { + using var volume = LoadVolume(name); + var info = NtfsJournal.Analyze(volume); + Assert.True(info.Status == expected, $"{name}: expected {expected}, got {info}"); + Assert.Equal("2.0", info.LogVersion); + + var bootSector = new byte[512]; + + DiscUtils.Ntfs.Journal.JournalUtilities.ReadExactly(volume, 0, bootSector, 0, 512); + Assert.Equal(clusterSize, BitConverter.ToUInt16(bootSector, 0x0B) * bootSector[0x0D]); + } + + [Theory] + [MemberData(nameof(Samples))] + public void WithoutReplay_TheVolumeIsOutOfDate(string name, NtfsJournalStatus expected, int clusterSize) + { + // Why replaying matters: as it is on disk, the volume differs from what Windows makes of it. + using var volume = LoadVolume(name); + var digest = LoadDigest(name); + int differences; + try + { + differences = WindowsReplayDigest.Check(volume, digest).Count(l => !l.StartsWith("file ", StringComparison.Ordinal)); + } + catch (IOException) + { + // Directories the stale metadata cannot even find (the index points at entries that are not there yet). + differences = int.MaxValue; + } + + Assert.True(expected == NtfsJournalStatus.NeedsReplay ? differences > 0 : differences == 0, $"{name}: {differences}"); + Assert.True(clusterSize > 0); + } + + [Theory] + [MemberData(nameof(Samples))] + public void Replay_GivesWhatWindowsReplayGives(string name, NtfsJournalStatus expected, int clusterSize) + { + using var volume = LoadVolume(name); + var info = NtfsJournal.Replay(volume); + Assert.True(info.Status == expected, $"{name}: expected {expected}, got {info}"); + Assert.True(info.Replayed); + + var mismatches = WindowsReplayDigest.Check(volume, LoadDigest(name)); + Assert.True(mismatches.Count == 0, $"{name}: {mismatches.Count} differences from Windows' replay, first: {mismatches.FirstOrDefault()}"); + Assert.Equal(NtfsJournalStatus.Clean, NtfsJournal.Analyze(volume).Status); + Assert.True(clusterSize > 0); + } + + [Theory] + [MemberData(nameof(Samples))] + public void ReadOnlyOpen_ShowsWhatWindowsShows_AndWritesNothing(string name, NtfsJournalStatus expected, int clusterSize) + { + using var volume = LoadVolume(name); + var image = volume.ToArray(); + using var readOnly = new MemoryStream(image, writable: false); + var info = NtfsJournal.ReplayInMemory(readOnly, out var replayed); + Assert.Equal(expected, info.Status); + using (replayed) + { + var mismatches = WindowsReplayDigest.Check(replayed ?? (Stream)readOnly, LoadDigest(name)); + Assert.True(mismatches.Count == 0, $"{name}: {mismatches.Count} differences, first: {mismatches.FirstOrDefault()}"); + } + + using (var ntfs = new NtfsFileSystem(new MemoryStream(image, writable: false))) + { + Assert.Equal(expected, ntfs.JournalInfo.Status); + Assert.NotEmpty(ntfs.GetDirectories(@"\")); + } + + Assert.True(clusterSize > 0); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsLogActionsRefusalTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsLogActionsRefusalTest.cs new file mode 100644 index 000000000..867917ee1 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsLogActionsRefusalTest.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// Every check an action makes on the page before changing it: each case breaks one of them, and the action must +/// refuse instead of writing. +/// +public partial class NtfsLogActionsTest +{ + private static readonly Dictionary> Malformed = new() + { + // Index buffers + ["delete: entry of length 0"] = () => + { + var a = Entry(1, "a"); + var q = IndexBuffer(a, LastEntry()); + Put16(q, 0x40 + a.Length + 8, 0); + return (NtfsLogOperation.DeleteIndexEntryAllocation, q, Record(NtfsLogOperation.DeleteIndexEntryAllocation, NtfsLogOperation.Noop, [], [], attributeOffset: 0x40 + a.Length), true); + }, + ["delete: entry past the used part"] = () => + { + var a = Entry(1, "a"); + var q = IndexBuffer(a, LastEntry()); + Put16(q, 0x40 + a.Length + 8, 0x800); + return (NtfsLogOperation.DeleteIndexEntryAllocation, q, Record(NtfsLogOperation.DeleteIndexEntryAllocation, NtfsLogOperation.Noop, [], [], attributeOffset: 0x40 + a.Length), true); + }, + ["write end: past the allocated size"] = () => (NtfsLogOperation.WriteEndOfIndexBuffer, IndexBuffer(LastEntry()), + Record(NtfsLogOperation.WriteEndOfIndexBuffer, NtfsLogOperation.Noop, new byte[4096], [], attributeOffset: 0x40), true), + ["write end: past the buffer"] = () => + { + var q = IndexBuffer(LastEntry()); + Put32(q, 0x20, 0x10000); + return (NtfsLogOperation.WriteEndOfIndexBuffer, q, Record(NtfsLogOperation.WriteEndOfIndexBuffer, NtfsLogOperation.Noop, new byte[4096], [], attributeOffset: 0x40), true); + }, + ["set vcn: data too short"] = () => (NtfsLogOperation.SetIndexEntryVcnAllocation, IndexBuffer(Entry(1, "a", 3), LastEntry()), + Record(NtfsLogOperation.SetIndexEntryVcnAllocation, NtfsLogOperation.Noop, new byte[4], [], attributeOffset: 0x40), true), + ["set vcn: entry too short"] = () => (NtfsLogOperation.SetIndexEntryVcnAllocation, IndexBuffer(LastEntry()), + Record(NtfsLogOperation.SetIndexEntryVcnAllocation, NtfsLogOperation.Noop, new byte[8], [], attributeOffset: 0x40), true), + ["set vcn: entry past the page"] = () => + { + var q = IndexBuffer(Entry(1, "a", 3), LastEntry()); + Put16(q, 0x40 + 8, 0xFF8); + return (NtfsLogOperation.SetIndexEntryVcnAllocation, q, Record(NtfsLogOperation.SetIndexEntryVcnAllocation, NtfsLogOperation.Noop, new byte[8], [], attributeOffset: 0x40), true); + }, + ["file name: entry too short"] = () => (NtfsLogOperation.UpdateFileNameAllocation, IndexBuffer(LastEntry()), + Record(NtfsLogOperation.UpdateFileNameAllocation, NtfsLogOperation.Noop, new byte[0x38], [], attributeOffset: 0x40), true), + ["file name: entry past the page"] = () => + { + var q = IndexBuffer(Entry(1, "a"), LastEntry()); + Put16(q, 0x40 + 8, 0xFF8); + return (NtfsLogOperation.UpdateFileNameAllocation, q, Record(NtfsLogOperation.UpdateFileNameAllocation, NtfsLogOperation.Noop, new byte[0x38], [], attributeOffset: 0x40), true); + }, + ["record data: past the entry"] = () => (NtfsLogOperation.UpdateRecordDataAllocation, IndexBuffer(ViewEntry([1, 2, 3, 4], [9, 9, 9, 9]), LastEntry()), + Record(NtfsLogOperation.UpdateRecordDataAllocation, NtfsLogOperation.Noop, new byte[64], [], attributeOffset: 0x40), true), + ["record data: entry past the page"] = () => + { + var q = IndexBuffer(ViewEntry([1, 2, 3, 4], [9, 9, 9, 9]), LastEntry()); + Put16(q, 0x40 + 8, 0xFF8); + return (NtfsLogOperation.UpdateRecordDataAllocation, q, Record(NtfsLogOperation.UpdateRecordDataAllocation, NtfsLogOperation.Noop, [1], [], attributeOffset: 0x40), true); + }, + ["buffer header: entries before the header"] = () => + { + var q = IndexBuffer(LastEntry()); + Put32(q, 0x18, 0); + return (NtfsLogOperation.DeleteIndexEntryAllocation, q, Record(NtfsLogOperation.DeleteIndexEntryAllocation, NtfsLogOperation.Noop, [], [], attributeOffset: 0x40), true); + }, + ["buffer header: used past the page"] = () => + { + var q = IndexBuffer(LastEntry()); + Put32(q, 0x1C, 0x2000); + return (NtfsLogOperation.DeleteIndexEntryAllocation, q, Record(NtfsLogOperation.DeleteIndexEntryAllocation, NtfsLogOperation.Noop, [], [], attributeOffset: 0x40), true); + }, + ["buffer entry: a target at the end of the page"] = () => + { + var q = IndexBuffer(); + Put32(q, 0x18, 4096 - 0x18); + Put32(q, 0x1C, 4096 - 0x18); + return (NtfsLogOperation.DeleteIndexEntryAllocation, q, Record(NtfsLogOperation.DeleteIndexEntryAllocation, NtfsLogOperation.Noop, [], [], attributeOffset: 4096), true); + }, + ["buffer entry: a truncated entry before the target"] = () => + { + var q = IndexBuffer(); + Put32(q, 0x18, 4096 - 0x18 - 4); + Put32(q, 0x1C, 4096 - 0x18); + return (NtfsLogOperation.AddIndexEntryAllocation, q, Record(NtfsLogOperation.AddIndexEntryAllocation, NtfsLogOperation.Noop, LastEntry(), [], attributeOffset: 4096), true); + }, + ["add: entry shorter than a header"] = () => (NtfsLogOperation.AddIndexEntryAllocation, IndexBuffer(LastEntry()), AddWithLength(0x08), true), + ["add: entry longer than its data"] = () => (NtfsLogOperation.AddIndexEntryAllocation, IndexBuffer(LastEntry()), AddWithLength(0x100), true), + ["add: entry length not aligned"] = () => (NtfsLogOperation.AddIndexEntryAllocation, IndexBuffer(LastEntry()), AddWithLength(0x1C), true), + + // File records + ["create: length shorter than a header"] = () => (NtfsLogOperation.CreateAttribute, FileRecord(Si()), CreateWithLength(0x10, 0x20), false), + ["create: length not aligned"] = () => (NtfsLogOperation.CreateAttribute, FileRecord(Si()), CreateWithLength(0x1C, 0x20), false), + ["create: length longer than the data"] = () => (NtfsLogOperation.CreateAttribute, FileRecord(Si()), CreateWithLength(0x100, 0x20), false), + ["mapping pairs: before the run list"] = () => (NtfsLogOperation.UpdateMappingPairs, FileRecord(Si(), Runs()), MappingPairs([0x11, 4, 0x20, 0], 0x10), false), + ["mapping pairs: past the attribute"] = () => (NtfsLogOperation.UpdateMappingPairs, FileRecord(Si(), Runs()), MappingPairs([0x11, 4, 0x20, 0], 0x200), false), + ["mapping pairs: a run with no length"] = () => (NtfsLogOperation.UpdateMappingPairs, FileRecord(Si(), Runs()), MappingPairs([0x10, 0x20, 0], 0x40), false), + ["mapping pairs: a run length of nine bytes"] = () => (NtfsLogOperation.UpdateMappingPairs, FileRecord(Si(), Runs()), MappingPairs([0x19, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0], 0x40), false), + ["mapping pairs: a run offset of nine bytes"] = () => (NtfsLogOperation.UpdateMappingPairs, FileRecord(Si(), Runs()), MappingPairs([0x91, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0], 0x40), false), + ["mapping pairs: a run past the attribute"] = () => (NtfsLogOperation.UpdateMappingPairs, FileRecord(Si(), Runs()), MappingPairs([0x11, 4, 0x20, 0x11, 4, 0x20, 0x11, 1], 0x40), false), + ["mapping pairs: a run that overflows the VCN"] = () => + { + var r = FileRecord(Si(), NonResident([0x11, 1, 0x20, 0], long.MaxValue - 1, long.MaxValue - 1, 1)); + return (NtfsLogOperation.UpdateMappingPairs, r, MappingPairs([0x11, 4, 0x20, 0], 0x40), false); + }, + ["new sizes: truncated non-resident header"] = () => + { + var r = FileRecord(Si(), Resident(0x80, [], 1)); + r[0x98 + 8] = 1; + return (NtfsLogOperation.SetNewAttributeSizes, r, Record(NtfsLogOperation.SetNewAttributeSizes, NtfsLogOperation.Noop, new byte[24], [], recordOffset: 0x98), false); + }, + ["root: delete an entry of length 0"] = () => + { + var r = FileRecord(Si(), IndexRoot(Entry(1, "a"), LastEntry())); + Put16(r, 0x98 + 0x38 + 8, 0); + return (NtfsLogOperation.DeleteIndexEntryRoot, r, Record(NtfsLogOperation.DeleteIndexEntryRoot, NtfsLogOperation.Noop, [], [], recordOffset: 0x98, attributeOffset: 0x38), false); + }, + ["root: delete an entry past the used part"] = () => + { + var r = FileRecord(Si(), IndexRoot(Entry(1, "a"), LastEntry())); + Put16(r, 0x98 + 0x38 + 8, 0x300); + return (NtfsLogOperation.DeleteIndexEntryRoot, r, Record(NtfsLogOperation.DeleteIndexEntryRoot, NtfsLogOperation.Noop, [], [], recordOffset: 0x98, attributeOffset: 0x38), false); + }, + ["root header: entries before the header"] = () => + { + var r = FileRecord(Si(), IndexRoot(LastEntry())); + Put32(r, 0x98 + 0x28, 0); + return (NtfsLogOperation.SetIndexEntryVcnRoot, r, Record(NtfsLogOperation.SetIndexEntryVcnRoot, NtfsLogOperation.Noop, new byte[8], [], recordOffset: 0x98, attributeOffset: 0x38), false); + }, + ["root header: used past the attribute"] = () => + { + var r = FileRecord(Si(), IndexRoot(LastEntry())); + Put32(r, 0x98 + 0x2C, 0x400); + return (NtfsLogOperation.SetIndexEntryVcnRoot, r, Record(NtfsLogOperation.SetIndexEntryVcnRoot, NtfsLogOperation.Noop, new byte[8], [], recordOffset: 0x98, attributeOffset: 0x38), false); + }, + ["root header: resident value offset past the file record"] = () => + { + var r = FileRecord(Si(), IndexRoot(LastEntry())); + Put16(r, 0x98 + 0x14, ushort.MaxValue); + return (NtfsLogOperation.SetIndexEntryVcnRoot, r, Record(NtfsLogOperation.SetIndexEntryVcnRoot, NtfsLogOperation.Noop, new byte[8], [], recordOffset: 0x98, attributeOffset: 0x38), false); + }, + ["record header: used past the record"] = () => + { + var r = FileRecord(Si()); + Put32(r, 0x18, 0x500); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x38), false); + }, + ["record header: attributes start in the header"] = () => + { + var r = FileRecord(Si()); + Put16(r, 0x14, 0x10); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x38), false); + }, + ["record header: attributes start past the used part"] = () => + { + var r = FileRecord(Si()); + Put16(r, 0x14, 0x300); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x38), false); + }, + ["attribute list: an attribute shorter than a header"] = () => + { + var r = FileRecord(Si(), Resident(0x80, [1], 1)); + Put32(r, 0x38 + 4, 0x10); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x98), false); + }, + ["attribute list: an attribute past the used part"] = () => + { + var r = FileRecord(Si(), Resident(0x80, [1], 1)); + Put32(r, 0x38 + 4, 0x300); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x98), false); + }, + ["attribute: the changed one is shorter than a header"] = () => + { + var r = FileRecord(Si(), Resident(0x80, [1], 1)); + Put32(r, 0x98 + 4, 0x10); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x98), false); + }, + ["attribute: the changed one runs past the used part"] = () => + { + var r = FileRecord(Si(), Resident(0x80, [1], 1)); + Put32(r, 0x98 + 4, 0x300); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x98), false); + }, + ["attribute: the length wraps around the record"] = () => + { + var r = FileRecord(Si(), Resident(0x80, [1], 1)); + Put32(r, 0x98 + 4, int.MaxValue - 7); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: 0x98), false); + }, + ["attribute: type fits in the record but length does not"] = () => + { + var r = FileRecord(Si()); + Put32(r, 0x38 + 4, RecordSize - 0x38 - 4); + Put32(r, RecordSize - 4, 0x80); + Put32(r, 0x18, RecordSize); + return (NtfsLogOperation.DeleteAttribute, r, Record(NtfsLogOperation.DeleteAttribute, NtfsLogOperation.Noop, [], [], recordOffset: RecordSize - 4), false); + }, + }; + + public static IEnumerable MalformedCases => Malformed.Keys.Select(k => new object[] { k }); + + private static byte[] Si() => Resident(0x10, new byte[0x48], 0); + + private static byte[] Runs() => NonResident([0x11, 0x04, 0x20, 0x00], 0, 3, 1); + + private static NtfsLogRecord AddWithLength(int length) + { + var entry = Entry(2, "b"); + Put16(entry, 8, (ushort)length); + return Record(NtfsLogOperation.AddIndexEntryAllocation, NtfsLogOperation.Noop, entry, [], attributeOffset: 0x40); + } + + private static NtfsLogRecord CreateWithLength(int length, int dataLength) + { + var data = new byte[dataLength]; + Put32(data, 0, 0x80); + Put32(data, 4, (uint)length); + return Record(NtfsLogOperation.CreateAttribute, NtfsLogOperation.Noop, data, [], recordOffset: 0x98); + } + + private static NtfsLogRecord MappingPairs(byte[] runs, int attributeOffset) => + Record(NtfsLogOperation.UpdateMappingPairs, NtfsLogOperation.Noop, runs, [], recordOffset: 0x98, attributeOffset: attributeOffset); + + [Theory] + [MemberData(nameof(MalformedCases))] + public void ActionsOnPagesThatDoNotMatchTheRecord_AreRefused(string name) + { + var (op, page, record, indexBuffer) = Malformed[name](); + var before = (byte[])page.Clone(); + Assert.Throws(() => + { + if (indexBuffer) + { + NtfsLogActions.ApplyToIndexBuffer(op, record, page, record.RedoData); + } + else + { + NtfsLogActions.ApplyToFileRecord(op, record, page, record.RedoData); + } + }); + + // Checks come before changes, apart from a run list found to be damaged after it was written. + if (op != NtfsLogOperation.UpdateMappingPairs) + { + Assert.True(before.AsSpan().SequenceEqual(page), name); + } + } + + [Fact] + public void ARunListEndingAtTheAttributeEnd_NeedsNoTerminator() + { + var page = FileRecord(Si(), Runs()); + var runs = new byte[] { 0x11, 4, 0x20, 0x11, 4, 0x20, 0x01, 0x01 }; + var record = MappingPairs(runs, 0x40); + NtfsLogActions.ApplyToFileRecord(record.RedoOperation, record, page, record.RedoData); + Assert.Equal(8UL, EndianUtilities.ToUInt64LittleEndian(page, 0x98 + 0x18)); + } +} diff --git a/Tests/LibraryTests/Ntfs/Journal/NtfsLogActionsTest.cs b/Tests/LibraryTests/Ntfs/Journal/NtfsLogActionsTest.cs new file mode 100644 index 000000000..edc8509d8 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/NtfsLogActionsTest.cs @@ -0,0 +1,608 @@ +using System; +using System.Collections.Generic; +using System.Text; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// The byte-level redo and undo actions, on file records and index buffers built here: every action applied and +/// then undone with the undo action NTFS logs for it must give back the original used bytes. +/// +public partial class NtfsLogActionsTest +{ + private const int RecordSize = 1024; + private const int FirstAttribute = 0x38; + + #region builders + + private static byte[] Resident(uint type, byte[] value, ushort id, bool indexed = false) + { + var length = (0x18 + value.Length + 7) & ~7; + var a = new byte[length]; + Put32(a, 0x00, type); + Put32(a, 0x04, (uint)length); + Put16(a, 0x0A, 0x18); + Put16(a, 0x0E, id); + Put32(a, 0x10, (uint)value.Length); + Put16(a, 0x14, 0x18); + a[0x16] = (byte)(indexed ? 1 : 0); + value.CopyTo(a, 0x18); + return a; + } + + private static byte[] NonResident(byte[] runs, long startVcn, long lastVcn, ushort id, bool withTotalSize = false) + { + var runsOffset = withTotalSize ? 0x48 : 0x40; + var length = (runsOffset + runs.Length + 7) & ~7; + var a = new byte[length]; + Put32(a, 0x00, 0x80); + Put32(a, 0x04, (uint)length); + a[0x08] = 1; + Put16(a, 0x0A, 0x40); + Put16(a, 0x0E, id); + Put64(a, 0x10, (ulong)startVcn); + Put64(a, 0x18, (ulong)lastVcn); + Put16(a, 0x20, (ushort)runsOffset); + Put64(a, 0x28, 0x10000); + Put64(a, 0x30, 0x8000); + Put64(a, 0x38, 0x8000); + runs.CopyTo(a, runsOffset); + return a; + } + + private static byte[] Entry(ulong fileReference, string name, long? childVcn = null) + { + var key = new byte[(0x42 + (2 * name.Length) + 7) & ~7]; + Put64(key, 0x00, 5); + for (var i = 0x08; i < 0x40; i++) + { + key[i] = (byte)(i + name.Length); + } + + key[0x40] = (byte)name.Length; + Encoding.Unicode.GetBytes(name).CopyTo(key, 0x42); + var length = 0x10 + key.Length + (childVcn.HasValue ? 8 : 0); + var e = new byte[length]; + Put64(e, 0x00, fileReference); + Put16(e, 0x08, (ushort)length); + Put16(e, 0x0A, (ushort)key.Length); + Put16(e, 0x0C, (ushort)(childVcn.HasValue ? 1 : 0)); + key.CopyTo(e, 0x10); + if (childVcn.HasValue) + { + Put64(e, length - 8, (ulong)childVcn.Value); + } + + return e; + } + + private static byte[] LastEntry(long? childVcn = null) + { + var length = 0x10 + (childVcn.HasValue ? 8 : 0); + var e = new byte[length]; + Put16(e, 0x08, (ushort)length); + Put16(e, 0x0C, (ushort)(2 | (childVcn.HasValue ? 1 : 0))); + if (childVcn.HasValue) + { + Put64(e, length - 8, (ulong)childVcn.Value); + } + + return e; + } + + private static byte[] ViewEntry(byte[] key, byte[] data) + { + var length = (0x10 + key.Length + data.Length + 7) & ~7; + var e = new byte[length]; + Put16(e, 0x00, (ushort)(0x10 + key.Length)); + Put16(e, 0x02, (ushort)data.Length); + Put16(e, 0x08, (ushort)length); + Put16(e, 0x0A, (ushort)key.Length); + key.CopyTo(e, 0x10); + data.CopyTo(e, 0x10 + key.Length); + return e; + } + + private static byte[] IndexRoot(params byte[][] entries) + { + var size = 0; + foreach (var e in entries) + { + size += e.Length; + } + + var value = new byte[0x20 + size]; + Put32(value, 0x00, 0x30); + Put32(value, 0x04, 1); + Put32(value, 0x08, 4096); + value[0x0C] = 1; + Put32(value, 0x10, 0x10); + Put32(value, 0x14, (uint)(0x10 + size)); + Put32(value, 0x18, (uint)(0x10 + size)); + var position = 0x20; + foreach (var e in entries) + { + e.CopyTo(value, position); + position += e.Length; + } + + return Resident(0x90, value, 4); + } + + /// A file record (logical form) holding the attributes in order, then the end marker. + private static byte[] FileRecord(params byte[][] attributes) + { + var r = new byte[RecordSize]; + Put32(r, 0x00, JournalUtilities.SignatureFile); + Put16(r, 0x04, 0x30); + Put16(r, 0x06, 3); + Put16(r, 0x10, 1); + Put16(r, 0x12, 1); + Put16(r, 0x14, FirstAttribute); + Put16(r, 0x16, 1); + Put32(r, 0x1C, RecordSize); + Put16(r, 0x28, 8); + var position = FirstAttribute; + foreach (var a in attributes) + { + a.CopyTo(r, position); + position += a.Length; + } + + Put32(r, position, 0xFFFFFFFF); + Put32(r, 0x18, (uint)(position + 8)); + return r; + } + + private static byte[] IndexBuffer(params byte[][] entries) + { + var q = new byte[4096]; + Put32(q, 0x00, JournalUtilities.SignatureIndx); + Put16(q, 0x04, 0x28); + Put16(q, 0x06, 9); + Put32(q, 0x18, 0x28); + var position = 0x40; + foreach (var e in entries) + { + e.CopyTo(q, position); + position += e.Length; + } + + Put32(q, 0x1C, (uint)(position - 0x18)); + Put32(q, 0x20, 4096 - 0x18); + return q; + } + + private static NtfsLogRecord Record(NtfsLogOperation redo, NtfsLogOperation undo, byte[] redoData, byte[] undoData, + int recordOffset = 0, int attributeOffset = 0) + { + var test = new TestLogRecord + { + Redo = redo, + Undo = undo, + RedoData = redoData, + UndoData = undoData, + RecordOffset = recordOffset, + AttributeOffset = attributeOffset, + Lcns = [100], + }; + var lfs = new LfsRecord { Lsn = 0x1000, RecordType = LfsRecord.TypeClientRecord, TransactionId = 0x18, ClientData = test.ClientData() }; + return NtfsLogRecord.TryParse(lfs, out var error) ?? throw new InvalidOperationException(error); + } + + private static int Used(byte[] r) => (int)EndianUtilities.ToUInt32LittleEndian(r, 0x18); + + private static void AssertUsedEqual(byte[] expected, byte[] actual) + { + Assert.Equal(Used(expected), Used(actual)); + Assert.True(expected.AsSpan(0, Used(expected)).SequenceEqual(actual.AsSpan(0, Used(actual)))); + } + + private static int IndxUsedEnd(byte[] q) => 0x18 + (int)EndianUtilities.ToUInt32LittleEndian(q, 0x1C); + + private static void AssertIndxEqual(byte[] expected, byte[] actual) + { + Assert.Equal(IndxUsedEnd(expected), IndxUsedEnd(actual)); + Assert.True(expected.AsSpan(0, IndxUsedEnd(expected)).SequenceEqual(actual.AsSpan(0, IndxUsedEnd(actual)))); + } + + private static void Redo(NtfsLogRecord record, byte[] page) => + NtfsLogActions.ApplyToFileRecord(record.RedoOperation, record, page, record.RedoData); + + private static void Undo(NtfsLogRecord record, byte[] page) => + NtfsLogActions.ApplyToFileRecord(record.UndoOperation, record, page, record.UndoData); + + #endregion + + [Fact] + public void CreateAndDeleteAttribute_RoundTrip_UpdateIdsAndHardLinks() + { + var si = Resident(0x10, new byte[0x48], 0); + var data = Resident(0x80, [1, 2, 3], 1); + var original = FileRecord(si, data); + var fileName = Resident(0x30, new byte[0x44], 9, indexed: true); + var create = Record(NtfsLogOperation.CreateAttribute, NtfsLogOperation.DeleteAttribute, fileName, [], + recordOffset: FirstAttribute + si.Length); + + var page = (byte[])original.Clone(); + Redo(create, page); + Assert.Equal(Used(original) + fileName.Length, Used(page)); + Assert.Equal(10, EndianUtilities.ToUInt16LittleEndian(page, 0x28)); + Assert.Equal(2, EndianUtilities.ToUInt16LittleEndian(page, 0x12)); + Assert.Equal(0x30u, EndianUtilities.ToUInt32LittleEndian(page, FirstAttribute + si.Length)); + + Undo(create, page); + Assert.Equal(1, EndianUtilities.ToUInt16LittleEndian(page, 0x12)); + Put16(page, 0x28, 8); + AssertUsedEqual(original, page); + } + + [Fact] + public void CreateAttribute_AtTheEndMarker_Appends() + { + var original = FileRecord(Resident(0x10, new byte[0x48], 0)); + var data = Resident(0x80, [7, 7], 1); + var create = Record(NtfsLogOperation.CreateAttribute, NtfsLogOperation.DeleteAttribute, data, [], + recordOffset: Used(original) - 8); + var page = (byte[])original.Clone(); + Redo(create, page); + Assert.Equal(0xFFFFFFFFu, EndianUtilities.ToUInt32LittleEndian(page, Used(page) - 8)); + Assert.Equal(0x80u, EndianUtilities.ToUInt32LittleEndian(page, Used(original) - 8)); + } + + [Theory] + [InlineData(new byte[] { 9, 9, 9 }, new byte[] { 1, 2, 3 })] + [InlineData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 }, new byte[] { 1, 2, 3 })] + [InlineData(new byte[] { 5 }, new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 })] + public void UpdateResidentValue_InPlaceGrowAndShrink_RoundTrip(byte[] newValue, byte[] oldValue) + { + var si = Resident(0x10, new byte[0x48], 0); + var data = Resident(0x80, oldValue, 1); + var after = Resident(0x30, new byte[0x44], 2, indexed: true); + var original = FileRecord(si, data, after); + var roff = FirstAttribute + si.Length; + var update = Record(NtfsLogOperation.UpdateResidentValue, NtfsLogOperation.UpdateResidentValue, newValue, oldValue, + recordOffset: roff, attributeOffset: 0x18); + + var page = (byte[])original.Clone(); + Redo(update, page); + Assert.True(page.AsSpan(roff + 0x18, newValue.Length).SequenceEqual(newValue)); + if (newValue.Length != oldValue.Length) + { + Assert.Equal((uint)newValue.Length, EndianUtilities.ToUInt32LittleEndian(page, roff + 0x10)); + } + + // The attribute after it follows the new size. + var next = roff + (int)EndianUtilities.ToUInt32LittleEndian(page, roff + 4); + Assert.Equal(0x30u, EndianUtilities.ToUInt32LittleEndian(page, next)); + + // Undone: the old value, length and layout. (Padding inside the attribute keeps bytes of the bigger + // value, as on Windows; it is not part of the value.) + Undo(update, page); + Assert.Equal(Used(original), Used(page)); + Assert.Equal(EndianUtilities.ToUInt32LittleEndian(original, roff + 4), EndianUtilities.ToUInt32LittleEndian(page, roff + 4)); + Assert.Equal((uint)oldValue.Length, EndianUtilities.ToUInt32LittleEndian(page, roff + 0x10)); + Assert.True(page.AsSpan(roff + 0x18, oldValue.Length).SequenceEqual(oldValue)); + var end = roff + (int)EndianUtilities.ToUInt32LittleEndian(page, roff + 4); + Assert.True(original.AsSpan(end, Used(original) - end).SequenceEqual(page.AsSpan(end, Used(page) - end))); + } + + [Fact] + public void UpdateMappingPairs_RewritesTheRunsAndTheLastVcn() + { + var si = Resident(0x10, new byte[0x48], 0); + var oldRuns = new byte[] { 0x11, 0x04, 0x20, 0x00 }; + var data = NonResident(oldRuns, 0, 3, 1); + var original = FileRecord(si, data); + var roff = FirstAttribute + si.Length; + + // Two runs (4 + 44 clusters) and the terminator. + var newRuns = new byte[] { 0x11, 0x04, 0x20, 0x21, 0x2C, 0x01, 0x40, 0x00 }; + var update = Record(NtfsLogOperation.UpdateMappingPairs, NtfsLogOperation.UpdateMappingPairs, newRuns, oldRuns, + recordOffset: roff, attributeOffset: 0x40); + var page = (byte[])original.Clone(); + Redo(update, page); + Assert.Equal(47UL, EndianUtilities.ToUInt64LittleEndian(page, roff + 0x18)); + Assert.True(page.AsSpan(roff + 0x40, newRuns.Length).SequenceEqual(newRuns)); + + // Undone: the old runs, last VCN and layout (bytes after the run list's terminator are not part of it). + Undo(update, page); + Assert.Equal(Used(original), Used(page)); + Assert.Equal(3UL, EndianUtilities.ToUInt64LittleEndian(page, roff + 0x18)); + Assert.True(page.AsSpan(roff + 0x40, oldRuns.Length).SequenceEqual(oldRuns)); + Assert.Equal(EndianUtilities.ToUInt32LittleEndian(original, roff + 4), EndianUtilities.ToUInt32LittleEndian(page, roff + 4)); + } + + [Fact] + public void SetNewAttributeSizes_SetsAllocatedValidAndDataSize_AndTotalSizeOnlyWhenTheAttributeHasIt() + { + var si = Resident(0x10, new byte[0x48], 0); + var runs = new byte[] { 0x11, 0x04, 0x20, 0x00 }; + foreach (var compressed in new[] { false, true }) + { + var page = FileRecord(si, NonResident(runs, 0, 3, 1, withTotalSize: compressed)); + var roff = FirstAttribute + si.Length; + var sizes = new byte[32]; + Put64(sizes, 0, 0x20000); + Put64(sizes, 8, 0x1100); + Put64(sizes, 16, 0x1234); + Put64(sizes, 24, 0x9000); + Redo(Record(NtfsLogOperation.SetNewAttributeSizes, NtfsLogOperation.SetNewAttributeSizes, sizes, sizes, recordOffset: roff), page); + Assert.Equal(0x20000UL, EndianUtilities.ToUInt64LittleEndian(page, roff + 0x28)); + Assert.Equal(0x1234UL, EndianUtilities.ToUInt64LittleEndian(page, roff + 0x30)); + Assert.Equal(0x1100UL, EndianUtilities.ToUInt64LittleEndian(page, roff + 0x38)); + Assert.Equal(compressed ? 0x9000UL : 0x11UL, EndianUtilities.ToUInt64LittleEndian(page, roff + 0x40) & (compressed ? ~0UL : 0xFFUL)); + } + } + + [Fact] + public void WriteEndOfFileRecordSegment_ReplacesTheTailAndSetsTheUsedSize() + { + var si = Resident(0x10, new byte[0x48], 0); + var page = FileRecord(si, Resident(0x80, new byte[100], 1)); + var tail = new byte[0x20]; + Put32(tail, 0x00, 0x80); + Put32(tail, 0x04, 0x18); + Put32(tail, 0x18, 0xFFFFFFFF); + var roff = FirstAttribute + si.Length; + Redo(Record(NtfsLogOperation.WriteEndOfFileRecordSegment, NtfsLogOperation.WriteEndOfFileRecordSegment, tail, [], recordOffset: roff), page); + Assert.Equal(roff + 0x20, Used(page)); + } + + [Fact] + public void InitializeDeallocateAndZeroEnd() + { + var image = FileRecord(Resident(0x10, new byte[0x48], 0)); + var page = new byte[RecordSize]; + page.AsSpan().Fill(0xAB); + var init = Record(NtfsLogOperation.InitializeFileRecordSegment, NtfsLogOperation.DeallocateFileRecordSegment, + image.AsSpan(0, Used(image)).ToArray(), []); + Redo(init, page); + AssertUsedEqual(image, page); + + Undo(init, page); + Assert.Equal(0, EndianUtilities.ToUInt16LittleEndian(page, 0x16) & 1); + Assert.Equal(2, EndianUtilities.ToUInt16LittleEndian(page, 0x10)); + + Redo(Record(NtfsLogOperation.ZeroEndOfFileRecord, NtfsLogOperation.Noop, new byte[0x100], [], recordOffset: 0x300), page); + Assert.True(AllAre(page.AsSpan(0x300, 0x100), 0)); + } + + [Fact] + public void IndexRootEntries_AddDeleteSetVcnAndUpdate() + { + var si = Resident(0x10, new byte[0x48], 0); + var first = Entry(0x0001000000000040, "a", childVcn: 2); + var root = IndexRoot(first, LastEntry(childVcn: 5)); + var original = FileRecord(si, root, Resident(0x80, [1], 1)); + var roff = FirstAttribute + si.Length; + var entriesStart = 0x18 + 0x20; + var added = Entry(0x0001000000000041, "bb", childVcn: 3); + + var add = Record(NtfsLogOperation.AddIndexEntryRoot, NtfsLogOperation.DeleteIndexEntryRoot, added, [], + recordOffset: roff, attributeOffset: entriesStart + first.Length); + var page = (byte[])original.Clone(); + Redo(add, page); + var attrLength = (int)EndianUtilities.ToUInt32LittleEndian(page, roff + 4); + Assert.Equal(root.Length + added.Length, attrLength); + Assert.Equal(0x80u, EndianUtilities.ToUInt32LittleEndian(page, roff + attrLength)); + + var vcn = new byte[8]; + Put64(vcn, 0, 77); + var setVcn = Record(NtfsLogOperation.SetIndexEntryVcnRoot, NtfsLogOperation.SetIndexEntryVcnRoot, vcn, [], + recordOffset: roff, attributeOffset: entriesStart + first.Length); + Redo(setVcn, page); + Assert.Equal(77UL, EndianUtilities.ToUInt64LittleEndian(page, roff + entriesStart + first.Length + added.Length - 8)); + + var info = new byte[0x38]; + info.AsSpan().Fill(0x5A); + var update = Record(NtfsLogOperation.UpdateFileNameRoot, NtfsLogOperation.UpdateFileNameRoot, info, [], + recordOffset: roff, attributeOffset: entriesStart); + Redo(update, page); + Assert.True(AllAre(page.AsSpan(roff + entriesStart + 0x18, 0x38), 0x5A)); + + Undo(add, page); + Assert.Equal(root.Length, (int)EndianUtilities.ToUInt32LittleEndian(page, roff + 4)); + Assert.Equal(Used(original), Used(page)); + } + + [Fact] + public void IndexRoot_UpdateRecordData_OfAViewIndex() + { + var si = Resident(0x10, new byte[0x48], 0); + var view = ViewEntry([1, 2, 3, 4], [9, 9, 9, 9, 9, 9, 9, 9]); + var root = IndexRoot(view, LastEntry()); + var page = FileRecord(si, root); + var roff = FirstAttribute + si.Length; + var update = Record(NtfsLogOperation.UpdateRecordDataRoot, NtfsLogOperation.UpdateRecordDataRoot, [4, 4, 4, 4, 4, 4, 4, 4], [], + recordOffset: roff, attributeOffset: 0x38); + Redo(update, page); + Assert.Equal(4, page[roff + 0x38 + 0x14]); + } + + [Fact] + public void IndexBufferEntries_AddDeleteWriteEndSetVcnAndUpdate() + { + var a = Entry(0x0001000000000040, "a"); + var b = Entry(0x0001000000000042, "c"); + var original = IndexBuffer(a, b, LastEntry()); + var added = Entry(0x0001000000000041, "b"); + var add = Record(NtfsLogOperation.AddIndexEntryAllocation, NtfsLogOperation.DeleteIndexEntryAllocation, added, [], + attributeOffset: 0x40 + a.Length); + var q = (byte[])original.Clone(); + NtfsLogActions.ApplyToIndexBuffer(add.RedoOperation, add, q, add.RedoData); + Assert.Equal(IndxUsedEnd(original) + added.Length, IndxUsedEnd(q)); + Assert.Equal(0x0001000000000041UL, EndianUtilities.ToUInt64LittleEndian(q, 0x40 + a.Length)); + + var info = new byte[0x38]; + info.AsSpan().Fill(0x33); + var update = Record(NtfsLogOperation.UpdateFileNameAllocation, NtfsLogOperation.UpdateFileNameAllocation, info, [], + attributeOffset: 0x40); + NtfsLogActions.ApplyToIndexBuffer(update.RedoOperation, update, q, update.RedoData); + Assert.Equal(0x33, q[0x40 + 0x18]); + + NtfsLogActions.ApplyToIndexBuffer(add.UndoOperation, add, q, add.UndoData); + q.AsSpan(0x40 + 0x18, 0x38).Clear(); + original.AsSpan(0x40 + 0x18, 0x38).Clear(); + AssertIndxEqual(original, q); + + // A split: the end of the buffer is rewritten, and an entry points to the new child buffer. + var tail = LastEntry(childVcn: 8); + var writeEnd = Record(NtfsLogOperation.WriteEndOfIndexBuffer, NtfsLogOperation.WriteEndOfIndexBuffer, tail, [], + attributeOffset: 0x40 + a.Length); + NtfsLogActions.ApplyToIndexBuffer(writeEnd.RedoOperation, writeEnd, q, writeEnd.RedoData); + Assert.Equal(0x40 + a.Length + tail.Length, IndxUsedEnd(q)); + + var vcn = new byte[8]; + Put64(vcn, 0, 12); + var setVcn = Record(NtfsLogOperation.SetIndexEntryVcnAllocation, NtfsLogOperation.SetIndexEntryVcnAllocation, vcn, [], + attributeOffset: 0x40 + a.Length); + NtfsLogActions.ApplyToIndexBuffer(setVcn.RedoOperation, setVcn, q, setVcn.RedoData); + Assert.Equal(12UL, EndianUtilities.ToUInt64LittleEndian(q, 0x40 + a.Length + tail.Length - 8)); + } + + [Fact] + public void IndexBuffer_UpdateRecordData_OfAViewIndex() + { + var view = ViewEntry([1, 2, 3, 4], [9, 9, 9, 9]); + var q = IndexBuffer(view, LastEntry()); + var update = Record(NtfsLogOperation.UpdateRecordDataAllocation, NtfsLogOperation.UpdateRecordDataAllocation, [6, 6, 6, 6], [], + attributeOffset: 0x40); + NtfsLogActions.ApplyToIndexBuffer(update.RedoOperation, update, q, update.RedoData); + Assert.Equal(6, q[0x40 + 0x14]); + } + + [Fact] + public void ChangeBits_SetsAndClearsRanges() + { + var bytes = new byte[4]; + NtfsLogActions.ChangeBits(bytes, 3, 10, set: true); + Assert.Equal(new byte[] { 0xF8, 0x1F, 0, 0 }, bytes); + NtfsLogActions.ChangeBits(bytes, 5, 4, set: false); + Assert.Equal(new byte[] { 0x18, 0x1E, 0, 0 }, bytes); + } + + public static IEnumerable RefusedFileRecordActions() + { + var si = Resident(0x10, new byte[0x48], 0); + var record = FileRecord(si, Resident(0x80, [1, 2], 1)); + var roff = FirstAttribute + si.Length; + + // No attribute starts there. + yield return [(int)NtfsLogOperation.DeleteAttribute, record, new byte[0], roff + 8, 0]; + + // The end marker is not an attribute to change. + yield return [(int)NtfsLogOperation.UpdateResidentValue, record, new byte[] { 1 }, Used(record) - 8, 0x18]; + + // The value update starts inside the attribute header. + yield return [(int)NtfsLogOperation.UpdateResidentValue, record, new byte[] { 1 }, roff, 0x08]; + + // An attribute that does not fit in the record. + var huge = Resident(0x80, new byte[1000], 3); + yield return [(int)NtfsLogOperation.CreateAttribute, record, huge, roff, 0]; + + // An index operation on an attribute that is not an index root. + yield return [(int)NtfsLogOperation.AddIndexEntryRoot, record, Entry(1, "x"), roff, 0x18]; + + // New sizes for a resident attribute. + yield return [(int)NtfsLogOperation.SetNewAttributeSizes, record, new byte[24], roff, 0]; + + // An operation that does not apply to file records. + yield return [(int)NtfsLogOperation.AddIndexEntryAllocation, record, new byte[16], roff, 0]; + } + + [Theory] + [MemberData(nameof(RefusedFileRecordActions))] + public void FileRecordActions_ThatDoNotFitTheRecord_AreRefused(int opCode, byte[] record, byte[] data, int roff, int aoff) + { + var op = (NtfsLogOperation)opCode; + var page = (byte[])record.Clone(); + var log = Record(op, NtfsLogOperation.Noop, data, [], recordOffset: roff, attributeOffset: aoff); + Assert.Throws(() => NtfsLogActions.ApplyToFileRecord(op, log, page, log.RedoData)); + } + + [Fact] + public void IndexBufferActions_ThatDoNotFitTheBuffer_AreRefused() + { + var a = Entry(0x0001000000000040, "a"); + var q = IndexBuffer(a, LastEntry()); + + // Not on an entry boundary. + var misplaced = Record(NtfsLogOperation.DeleteIndexEntryAllocation, NtfsLogOperation.Noop, [], [], attributeOffset: 0x48); + Assert.Throws(() => NtfsLogActions.ApplyToIndexBuffer(misplaced.RedoOperation, misplaced, q, misplaced.RedoData)); + + // Past the used part. + var beyond = Record(NtfsLogOperation.SetIndexEntryVcnAllocation, NtfsLogOperation.Noop, new byte[8], [], attributeOffset: 0x800); + Assert.Throws(() => NtfsLogActions.ApplyToIndexBuffer(beyond.RedoOperation, beyond, q, beyond.RedoData)); + + // An entry that does not fit any more. + var full = (byte[])q.Clone(); + Put32(full, 0x20, (uint)(IndxUsedEnd(full) - 0x18)); + var add = Record(NtfsLogOperation.AddIndexEntryAllocation, NtfsLogOperation.Noop, Entry(2, "b"), [], attributeOffset: 0x40); + Assert.Throws(() => NtfsLogActions.ApplyToIndexBuffer(add.RedoOperation, add, full, add.RedoData)); + + // A file record operation. + var wrong = Record(NtfsLogOperation.CreateAttribute, NtfsLogOperation.Noop, [], [], attributeOffset: 0x40); + Assert.Throws(() => NtfsLogActions.ApplyToIndexBuffer(wrong.RedoOperation, wrong, q, wrong.RedoData)); + } + + [Fact] + public void RecordData_LeftOutOfTheRecord_CountsAsZeros() + { + var test = new TestLogRecord + { + Redo = NtfsLogOperation.UpdateResidentValue, + Undo = NtfsLogOperation.UpdateResidentValue, + RedoData = [], + RedoLength = 12, + UndoData = [], + UndoLength = 0, + Lcns = [1], + }; + var lfs = new LfsRecord { Lsn = 1, RecordType = LfsRecord.TypeClientRecord, TransactionId = 0x18, ClientData = test.ClientData() }; + var record = NtfsLogRecord.TryParse(lfs, out _); + Assert.Equal(12, record.RedoData.Count); + Assert.True(AllAre(record.RedoData.AsSpan(), 0)); + } + + [Fact] + public void Records_WithMisplacedData_AreRejected() + { + var test = new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, RedoData = [1, 2, 3, 4, 5, 6, 7, 8] }; + var data = test.ClientData(); + Put16(data, 0x04, 0x22); + var lfs = new LfsRecord { Lsn = 1, RecordType = LfsRecord.TypeClientRecord, TransactionId = 0x18, ClientData = data }; + Assert.Null(NtfsLogRecord.TryParse(lfs, out var error)); + Assert.NotNull(error); + + var tooShort = new LfsRecord { Lsn = 1, RecordType = LfsRecord.TypeClientRecord, ClientData = new byte[8] }; + Assert.Null(NtfsLogRecord.TryParse(tooShort, out _)); + + var lcns = new byte[0x20]; + Put16(lcns, 0x0E, 3); + Assert.Null(NtfsLogRecord.TryParse(new LfsRecord { Lsn = 1, ClientData = lcns }, out _)); + } + + private static bool AllAre(ReadOnlySpan bytes, byte value) + { + foreach (var b in bytes) + { + if (b != value) + { + return false; + } + } + + return true; + } + + private static bool AllAre(ArraySegment bytes, byte value) => AllAre(bytes.AsSpan(), value); + + private static void Put16(byte[] b, int o, ushort v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); + + private static void Put32(byte[] b, int o, uint v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); + + private static void Put64(byte[] b, int o, ulong v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); +} diff --git a/Tests/LibraryTests/Ntfs/Journal/TestJournalWriter.cs b/Tests/LibraryTests/Ntfs/Journal/TestJournalWriter.cs new file mode 100644 index 000000000..3ebe26927 --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/TestJournalWriter.cs @@ -0,0 +1,492 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using DiscUtils.Ntfs.Journal; +using DiscUtils.Streams; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// One NTFS log record for . +/// +internal sealed class TestLogRecord +{ + public NtfsLogOperation Redo { get; set; } + + public NtfsLogOperation Undo { get; set; } + + public uint TransactionId { get; set; } = 0x18; + + public byte[] RedoData { get; set; } = []; + + public byte[] UndoData { get; set; } = []; + + /// Declared redo length when bytes are left out (Windows' trick for zeros); default: the data length. + public int? RedoLength { get; set; } + + public int? UndoLength { get; set; } + + public ushort TargetAttribute { get; set; } + + public long TargetVcn { get; set; } + + public int ClusterBlockOffset { get; set; } + + public int RecordOffset { get; set; } + + public int AttributeOffset { get; set; } + + public long[] Lcns { get; set; } = []; + + /// The previous record of this transaction to undo; 0 = none. + public ulong UndoNextLsn { get; set; } + + /// The NTFS client data: the 0x20-byte header, the clusters, then redo and undo data (8-byte aligned). + public byte[] ClientData() + { + var redoOffset = 0x20 + (8 * Lcns.Length); + var undoOffset = redoOffset + ((RedoData.Length + 7) & ~7); + var length = undoOffset + UndoData.Length; + var b = new byte[length]; + Put16(b, 0x00, (ushort)Redo); + Put16(b, 0x02, (ushort)Undo); + Put16(b, 0x04, (ushort)redoOffset); + Put16(b, 0x06, (ushort)(RedoLength ?? RedoData.Length)); + Put16(b, 0x08, (ushort)undoOffset); + Put16(b, 0x0A, (ushort)(UndoLength ?? UndoData.Length)); + Put16(b, 0x0C, TargetAttribute); + Put16(b, 0x0E, (ushort)Lcns.Length); + Put16(b, 0x10, (ushort)RecordOffset); + Put16(b, 0x12, (ushort)AttributeOffset); + Put16(b, 0x14, (ushort)ClusterBlockOffset); + Put64(b, 0x18, (ulong)TargetVcn); + for (var i = 0; i < Lcns.Length; i++) + { + Put64(b, 0x20 + (8 * i), (ulong)Lcns[i]); + } + + Array.Copy(RedoData, 0, b, redoOffset, RedoData.Length); + Array.Copy(UndoData, 0, b, undoOffset, UndoData.Length); + return b; + } + + private static void Put16(byte[] b, int o, ushort v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); + + private static void Put64(byte[] b, int o, ulong v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); +} + +/// +/// Writes a journal into a volume's $LogFile the way Windows' log file service lays it out: restart pages, record +/// pages with update sequence arrays, 8-byte aligned records continued after the next page's header, wrapping to +/// the first record page with the next sequence number, and (version 2.0) tail copies of recently written pages. +/// Tests use it to make a volume "Windows crashed on". +/// +internal sealed class TestJournalWriter +{ + public const int PageSize = 4096; + private const int DataOffset = 0x40; + private const int HeaderLength = 0x30; + + private readonly LogFileStorage _storage; + private readonly Dictionary _pages = []; + private readonly Dictionary _pageInfo = []; + private readonly Dictionary<(ushort Index, ushort Sequence), ulong> _lastClientLsns = []; + private long _position; + private ulong _sequence; + + public TestJournalWriter(Stream volume, int majorVersion = 2, long? startOffset = null, ulong sequence = 1) + { + Volume = volume; + Layout = JournalVolumeLayout.Read(volume); + _storage = LogFileStorage.Open(volume, Layout); + MajorVersion = majorVersion; + FileSize = _storage.Length & ~(long)(PageSize - 1); + var log2 = 0; + while ((1L << log2) < FileSize) + { + log2++; + } + + SequenceNumberBits = 66 - log2; + FirstPageOffset = (majorVersion >= 2 ? 0x22L : 4L) * PageSize; + _position = (startOffset ?? FirstPageOffset) + DataOffset; + _sequence = sequence; + } + + public Stream Volume { get; } + + public JournalVolumeLayout Layout { get; } + + public int MajorVersion { get; } + + public long FileSize { get; } + + public int SequenceNumberBits { get; } + + public long FirstPageOffset { get; } + + public ulong LastLsn { get; private set; } + + public ushort ClientSequenceNumber { get; set; } + + /// Where the next record goes (file offset in $LogFile). + public long Position => _position; + + /// Pages written to their place in the log (false: only to the tail copy area, as if the crash came first). + public HashSet PagesOnlyInTail { get; } = []; + + public ulong LsnOf(long offset, ulong sequence) => (sequence << (64 - SequenceNumberBits)) + (ulong)(offset >> 3); + + public long LsnToOffsetForTest(ulong lsn) => (long)((lsn << SequenceNumberBits) >> (SequenceNumberBits - 3)); + + /// Appends an NTFS client record and returns its LSN. + public ulong Append(TestLogRecord record, ushort flags = 0) => + AppendRaw(record.ClientData(), LfsRecord.TypeClientRecord, record.TransactionId, record.UndoNextLsn, flags); + + /// Appends any LFS record (the client data as given) and returns its LSN. + public ulong AppendRaw(byte[] clientData, uint recordType, uint transactionId, ulong undoNextLsn, ushort flags = 0, + ushort? clientSequence = null, ushort clientIndex = 0) + { + // A record starts after the page header, and on the next page when its header no longer fits. + if (_position % PageSize == 0 || PageSize - (_position % PageSize) < HeaderLength) + { + NextPage(); + } + + var lsn = LsnOf(_position, _sequence); + var clientId = (Index: clientIndex, Sequence: clientSequence ?? ClientSequenceNumber); + _lastClientLsns.TryGetValue(clientId, out var previousLsn); + var header = new byte[HeaderLength]; + Put64(header, 0x00, lsn); + Put64(header, 0x08, previousLsn); + Put64(header, 0x10, undoNextLsn); + Put32(header, 0x18, (uint)clientData.Length); + Put16(header, 0x1C, clientId.Sequence); + Put16(header, 0x1E, clientIndex); + Put32(header, 0x20, recordType); + Put32(header, 0x24, transactionId); + var multiPage = (_position % PageSize) + HeaderLength + clientData.Length > PageSize; + Put16(header, 0x28, (ushort)(flags | (multiPage ? 1 : 0))); + + var bytes = new byte[HeaderLength + clientData.Length]; + Array.Copy(header, bytes, HeaderLength); + Array.Copy(clientData, 0, bytes, HeaderLength, clientData.Length); + + var written = 0; + while (written < bytes.Length) + { + var page = Page(_position & ~(long)(PageSize - 1)); + var inPage = (int)(_position % PageSize); + var chunk = Math.Min(bytes.Length - written, PageSize - inPage); + Array.Copy(bytes, written, page, inPage, chunk); + written += chunk; + _position += chunk; + var info = _pageInfo[_position - 1 & ~(long)(PageSize - 1)]; + info.LastLsn = lsn; + if (written == bytes.Length) + { + info.LastEndLsn = lsn; + info.RecordEnds = true; + info.NextFree = (int)(((_position + 7) & ~7L) % PageSize); + } + + _pageInfo[_position - 1 & ~(long)(PageSize - 1)] = info; + if (written < bytes.Length) + { + NextPage(); + } + } + + _position = (_position + 7) & ~7L; + _lastClientLsns[clientId] = lsn; + LastLsn = lsn; + return lsn; + } + + /// + /// Writes a checkpoint: the table dumps and the NTFS restart record. Returns the restart record's LSN (what the + /// restart area's client points at). + /// + public ulong Checkpoint( + IReadOnlyList<(ushort Offset, uint Type, ulong FileReference, uint BytesPerIndex, string Name)> openAttributes, + IReadOnlyList<(uint TargetAttribute, long Vcn, ulong OldestLsn, long[] Lcns)>? dirtyPages = null, + IReadOnlyList<(uint Offset, TransactionState State, ulong FirstLsn, ulong UndoNextLsn)>? transactions = null, + int clustersPerPage = 1, + ulong? checkpointLsn = null, + uint ntfsVersion = 1) + { + var start = Append(new TestLogRecord { Redo = NtfsLogOperation.Noop, Undo = NtfsLogOperation.Noop, TransactionId = 0x18 }); + + ulong oatLsn = 0, namesLsn = 0, dptLsn = 0, ttLsn = 0; + int oatLen = 0, namesLen = 0, dptLen = 0, ttLen = 0; + if (openAttributes is { Count: > 0 }) + { + var max = 0; + foreach (var a in openAttributes) + { + max = Math.Max(max, a.Offset); + } + + var entrySize = ntfsVersion == 0 ? 0x2C : 0x28; + var table = Table(entrySize, (max - 0x18) / entrySize + 1); + var names = new MemoryStream(); + foreach (var a in openAttributes) + { + Put32(table, a.Offset, 0xFFFFFFFF); + OpenAttributeEntryBytes(a.Type, a.FileReference, a.BytesPerIndex, ntfsVersion).AsSpan(4).CopyTo(table.AsSpan(a.Offset + 4)); + if (!string.IsNullOrEmpty(a.Name)) + { + var name = Encoding.Unicode.GetBytes(a.Name); + names.Write(BitConverter.GetBytes(a.Offset), 0, 2); + names.Write(BitConverter.GetBytes((ushort)name.Length), 0, 2); + names.Write(name, 0, name.Length); + names.Write(new byte[2], 0, 2); + } + } + + oatLen = table.Length; + oatLsn = Append(Dump(NtfsLogOperation.OpenAttributeTableDump, table)); + if (names.Length > 0) + { + names.Write(new byte[4], 0, 4); + var bytes = names.ToArray(); + namesLen = bytes.Length; + namesLsn = Append(Dump(NtfsLogOperation.AttributeNamesDump, bytes)); + } + } + + if (dirtyPages is { Count: > 0 }) + { + var lcnsOffset = ntfsVersion == 0 ? 0x24 : 0x20; + var size = lcnsOffset + (8 * clustersPerPage); + var table = Table(size, dirtyPages.Count); + for (var i = 0; i < dirtyPages.Count; i++) + { + var e = 0x18 + (i * size); + var page = dirtyPages[i]; + Put32(table, e, 0xFFFFFFFF); + Put32(table, e + 0x04, page.TargetAttribute); + Put32(table, e + 0x08, (uint)(clustersPerPage * Layout.BytesPerCluster)); + Put32(table, e + 0x0C, (uint)clustersPerPage); + Put64(table, e + (ntfsVersion == 0 ? 0x14 : 0x10), (ulong)page.Vcn); + Put64(table, e + (ntfsVersion == 0 ? 0x1C : 0x18), page.OldestLsn); + for (var j = 0; j < page.Lcns.Length; j++) + { + Put64(table, e + lcnsOffset + (8 * j), (ulong)page.Lcns[j]); + } + } + + dptLen = table.Length; + dptLsn = Append(Dump(NtfsLogOperation.DirtyPageTableDump, table)); + } + + if (transactions is { Count: > 0 }) + { + var max = 0u; + foreach (var t in transactions) + { + max = Math.Max(max, t.Offset); + } + + var table = Table(0x28, (int)((max - 0x18) / 0x28) + 1); + foreach (var t in transactions) + { + var e = (int)t.Offset; + Put32(table, e, 0xFFFFFFFF); + table[e + 4] = (byte)t.State; + Put64(table, e + 0x08, t.FirstLsn); + Put64(table, e + 0x10, t.UndoNextLsn); + Put64(table, e + 0x18, t.UndoNextLsn); + } + + ttLen = table.Length; + ttLsn = Append(Dump(NtfsLogOperation.TransactionTableDump, table)); + } + + var restart = new byte[0x40]; + Put32(restart, 0x00, ntfsVersion); + Put32(restart, 0x04, 0); + Put64(restart, 0x08, checkpointLsn ?? start); + Put64(restart, 0x10, oatLsn); + Put64(restart, 0x18, namesLsn); + Put64(restart, 0x20, dptLsn); + Put64(restart, 0x28, ttLsn); + Put32(restart, 0x30, (uint)oatLen); + Put32(restart, 0x34, (uint)namesLen); + Put32(restart, 0x38, (uint)dptLen); + Put32(restart, 0x3C, (uint)ttLen); + return AppendRaw(restart, LfsRecord.TypeClientRestart, 0, 0); + } + + /// + /// Writes the record pages and both restart pages. is the NTFS restart record. + /// + public void Finish(ulong restartLsn, ulong oldestLsn = 0, bool clean = false, ulong? currentLsn = null) + { + var tailSlot = 0; + foreach (var pair in _pages) + { + var page = (byte[])pair.Value.Clone(); + var info = _pageInfo[pair.Key]; + Put32(page, 0, JournalUtilities.SignatureRcrd); + Put16(page, 4, 0x28); + Put16(page, 6, PageSize / 512 + 1); + Put64(page, 0x08, MajorVersion >= 2 ? info.LastLsn : info.LastLsn); + Put32(page, 0x10, info.RecordEnds ? 1u : 0u); + Put16(page, 0x14, 1); + Put16(page, 0x16, 1); + Put16(page, 0x18, (ushort)info.NextFree); + Put64(page, 0x20, info.LastEndLsn); + if (MajorVersion >= 2) + { + Put32(page, 0x3C, (uint)pair.Key); + } + + JournalUtilities.ApplyFixups(page, 0, PageSize); + if (!PagesOnlyInTail.Contains(pair.Key)) + { + _storage.Write(pair.Key, page, 0, PageSize); + } + + if (PagesOnlyInTail.Contains(pair.Key) || TailCopies) + { + var copy = (byte[])pair.Value.Clone(); + Array.Copy(page, copy, PageSize); + JournalUtilities.RemoveFixups(copy, 0, PageSize); + if (MajorVersion < 2) + { + // Version 1.x tail copies keep the page's place where an LSN would be. + Put64(copy, 0x08, (ulong)pair.Key); + } + + JournalUtilities.ApplyFixups(copy, 0, PageSize); + var slot = 2 + (tailSlot++ % (MajorVersion >= 2 ? 0x20 : 2)); + _storage.Write((long)slot * PageSize, copy, 0, PageSize); + } + } + + var restart = RestartPage(restartLsn, oldestLsn, clean, currentLsn ?? LastLsn); + _storage.Write(0, restart, 0, PageSize); + _storage.Write(PageSize, restart, 0, PageSize); + Volume.Flush(); + } + + /// Whether every record page is also copied into the tail area. + public bool TailCopies { get; set; } + + public byte[] RestartPage(ulong restartLsn, ulong oldestLsn, bool clean, ulong currentLsn) + { + var page = new byte[PageSize]; + Put32(page, 0, JournalUtilities.SignatureRstr); + Put16(page, 4, 0x1E); + Put16(page, 6, PageSize / 512 + 1); + Put32(page, 0x10, PageSize); + Put32(page, 0x14, PageSize); + Put16(page, 0x18, 0x30); + Put16(page, 0x1A, (ushort)(MajorVersion >= 2 ? 0 : 1)); + Put16(page, 0x1C, (ushort)MajorVersion); + const int ra = 0x30; + Put64(page, ra + 0x00, currentLsn); + Put16(page, ra + 0x08, 1); + Put16(page, ra + 0x0A, 0xFFFF); + Put16(page, ra + 0x0C, 0); + Put16(page, ra + 0x0E, (ushort)(clean ? 2 : 0)); + Put32(page, ra + 0x10, (uint)SequenceNumberBits); + Put16(page, ra + 0x14, 0xD0); + Put16(page, ra + 0x16, 0x40); + Put64(page, ra + 0x18, (ulong)FileSize); + Put16(page, ra + 0x24, HeaderLength); + Put16(page, ra + 0x26, DataOffset); + Put32(page, ra + 0x28, 7); + const int client = ra + 0x40; + Put64(page, client + 0x00, oldestLsn); + Put64(page, client + 0x08, restartLsn); + Put16(page, client + 0x10, 0xFFFF); + Put16(page, client + 0x12, 0xFFFF); + Put16(page, client + 0x14, ClientSequenceNumber); + Put32(page, client + 0x1C, 8); + Encoding.Unicode.GetBytes("NTFS").CopyTo(page, client + 0x20); + JournalUtilities.ApplyFixups(page, 0, PageSize); + return page; + } + + /// Fills the whole $LogFile with 0xFF, like a freshly formatted or reset log. + public void Erase() + { + var ff = new byte[PageSize]; + ff.AsSpan().Fill(0xFF); + for (long offset = 0; offset < FileSize; offset += PageSize) + { + _storage.Write(offset, ff, 0, PageSize); + } + } + + public LogFileStorage Storage => _storage; + + /// An open attribute table entry in the layout of NTFS restart version 0 or 1 (also the redo data of OpenNonresidentAttribute). + public static byte[] OpenAttributeEntryBytes(uint type, ulong fileReference, uint bytesPerIndex, uint ntfsVersion = 1) + { + var e = new byte[ntfsVersion == 0 ? 0x2C : 0x28]; + Put32(e, 0, 0xFFFFFFFF); + if (ntfsVersion == 0) + { + Put64(e, 0x08, fileReference); + Put32(e, 0x1C, type); + Put32(e, 0x28, bytesPerIndex); + } + else + { + Put32(e, 0x04, bytesPerIndex); + Put32(e, 0x08, type); + Put64(e, 0x10, fileReference); + } + + return e; + } + + private static TestLogRecord Dump(NtfsLogOperation op, byte[] table) => + new() { Redo = op, Undo = NtfsLogOperation.Noop, TransactionId = 0x18, RedoData = table }; + + private static byte[] Table(int entrySize, int entries) + { + var table = new byte[0x18 + (entrySize * entries)]; + Put16(table, 0x00, (ushort)entrySize); + Put16(table, 0x02, (ushort)entries); + Put16(table, 0x04, (ushort)entries); + return table; + } + + private byte[] Page(long offset) + { + if (!_pages.TryGetValue(offset, out var page)) + { + page = new byte[PageSize]; + _pages[offset] = page; + _pageInfo[offset] = (0, 0, DataOffset, false); + } + + return page; + } + + private void NextPage() + { + // The page the last byte went to (the position may already be at the start of the next one). + var next = ((_position - 1) & ~(long)(PageSize - 1)) + PageSize; + if (next >= FileSize) + { + next = FirstPageOffset; + _sequence++; + } + + _position = next + DataOffset; + } + + private static void Put16(byte[] b, int o, ushort v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); + + private static void Put16(byte[] b, int o, int v) => EndianUtilities.WriteBytesLittleEndian((ushort)v, b, o); + + private static void Put32(byte[] b, int o, uint v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); + + private static void Put64(byte[] b, int o, ulong v) => EndianUtilities.WriteBytesLittleEndian(v, b, o); +} diff --git a/Tests/LibraryTests/Ntfs/Journal/WindowsReplayDigest.cs b/Tests/LibraryTests/Ntfs/Journal/WindowsReplayDigest.cs new file mode 100644 index 000000000..8d7f17d4c --- /dev/null +++ b/Tests/LibraryTests/Ntfs/Journal/WindowsReplayDigest.cs @@ -0,0 +1,443 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using DiscUtils.Ntfs; + +namespace LibraryTests.Ntfs.Journal; + +/// +/// A compact description of what Windows' own journal replay made of a crashed volume, small enough to check in +/// next to the crashed volume: hashes of the used part of every MFT record and index buffer (LSN and update +/// sequence number left out, which every writer sets for itself), both bitmaps, and the file listing. Pages Windows +/// changed after its replay (mount-time housekeeping such as purging $Extend\$Deleted, which carries an LSN past +/// the crash's last record) are left out, with the bitmap bits they own. +/// +/// Also compiled into the lab tool that produces the samples, so both compute the same lines. +public static class WindowsReplayDigest +{ + /// + /// The digest of (Windows' replay of the crash), leaving out what Windows changed + /// after its replay. (DiscUtils' replay) adds the clusters those records own there. + /// + public static List Create(Stream windowsVolume, Stream replayedVolume, ulong lastLsn) + { + var windows = new Volume(windowsVolume); + var ours = new Volume(replayedVolume); + + // $Extend\$RmMetadata\$TxfLog (record 30) and System Volume Information: Windows writes into them at mount + // and refreshes their timestamps without logging. + var excludedRecords = new SortedSet { 30 }; + // $Extend\$Quota too: Windows charges the files it creates at mount to their owner without logging. + foreach (var path in new[] { @"\System Volume Information", @"\$Extend\$Quota" }) + { + if (windows.RecordOf(path) is { } record) + { + excludedRecords.Add(record); + } + } + + for (long n = 0; n < windows.RecordCount; n++) + { + if (windows.RecordLsn(n) > lastLsn) + { + excludedRecords.Add(n); + } + } + + var excludedClusters = new SortedSet(); + foreach (var volume in new[] { windows, ours }) + { + foreach (var pair in volume.ClusterOwners()) + { + if (excludedRecords.Contains(pair.Value)) + { + excludedClusters.Add(pair.Key); + } + } + } + + var lines = new List + { + $"excluded-records {string.Join(",", excludedRecords)}", + $"excluded-clusters {string.Join(",", excludedClusters)}", + }; + + for (long n = 0; n < windows.RecordCount; n++) + { + if (!excludedRecords.Contains(n)) + { + lines.Add($"mft {n} {windows.RecordHash(n)}"); + } + } + + lines.Add($"mft-bitmap {windows.MftBitmapHash(excludedRecords)}"); + lines.Add($"cluster-bitmap {windows.ClusterBitmapHash(excludedClusters)}"); + foreach (var (directory, index, hash, lsn) in windows.IndexBuffers(excludedRecords)) + { + if (lsn <= lastLsn) + { + lines.Add($"indx {directory}|{index} {hash}"); + } + } + + lines.AddRange(windows.Listing().Select(l => "file " + l)); + return lines; + } + + /// The lines of that does not match. + public static List Check(Stream replayedVolume, IReadOnlyList digest) + { + var ours = new Volume(replayedVolume); + var excludedRecords = Numbers(digest.First(l => l.StartsWith("excluded-records ", StringComparison.Ordinal))); + var excludedClusters = Numbers(digest.First(l => l.StartsWith("excluded-clusters ", StringComparison.Ordinal))); + var indx = ours.IndexBuffers(excludedRecords).ToDictionary(b => $"{b.Directory}|{b.Index}", b => b.Hash); + var listing = new HashSet(ours.Listing().Select(l => "file " + l)); + var expectedFiles = 0; + var mismatches = new List(); + foreach (var line in digest) + { + var parts = line.Split(' '); + switch (parts[0]) + { + case "mft": + var n = long.Parse(parts[1]); + if (n >= ours.RecordCount || ours.RecordHash(n) != parts[2]) + { + mismatches.Add(line); + } + + break; + case "mft-bitmap": + if (ours.MftBitmapHash(excludedRecords) != parts[1]) + { + mismatches.Add(line); + } + + break; + case "cluster-bitmap": + if (ours.ClusterBitmapHash(excludedClusters) != parts[1]) + { + mismatches.Add(line); + } + + break; + case "indx": + var key = line.Substring(5, line.LastIndexOf(' ') - 5); + if (!indx.TryGetValue(key, out var hash) || hash != parts[parts.Length - 1]) + { + mismatches.Add(line); + } + + break; + case "file": + expectedFiles++; + if (!listing.Contains(line)) + { + mismatches.Add(line); + } + + break; + } + } + + if (listing.Count != expectedFiles) + { + mismatches.Add($"listing has {listing.Count} entries, Windows {expectedFiles}"); + } + + return mismatches; + } + + private static SortedSet Numbers(string line) + { + var list = line.Substring(line.IndexOf(' ') + 1); + return new SortedSet(list.Length == 0 ? Enumerable.Empty() : list.Split(',').Select(long.Parse)); + } + + private static string Hash(byte[] bytes, int offset, int count) + { + using var sha = SHA256.Create(); + return BitConverter.ToString(sha.ComputeHash(bytes, offset, count)).Replace("-", "").Substring(0, 16); + } + + /// A clean NTFS volume read through DiscUtils' public API (journal ignored). + private sealed class Volume + { + private readonly NtfsFileSystem _fs; + private readonly byte[] _mft; + private readonly int _recordSize; + private readonly int _indexSize; + + public Volume(Stream source) + { + // Read-only: DiscUtils updates access times as it reads, which would change what is being measured. + source.Position = 0; + var copy = new MemoryStream(); + source.CopyTo(copy); + var stream = new MemoryStream(copy.ToArray(), writable: false); + var boot = new byte[512]; + for (var done = 0; done < boot.Length;) + { + done += stream.Read(boot, done, boot.Length - done); + } + + var cluster = BitConverter.ToUInt16(boot, 0x0B) * boot[0x0D]; + _recordSize = Size((sbyte)boot[0x40], cluster); + _indexSize = Size((sbyte)boot[0x44], cluster); + _fs = new NtfsFileSystem(stream, NtfsJournalMode.Ignore); + _fs.NtfsOptions.HideHiddenFiles = false; + _fs.NtfsOptions.HideSystemFiles = false; + _mft = ReadAll("$MFT"); + } + + public long RecordCount => _mft.Length / _recordSize; + + public long? RecordOf(string path) + { + // Metafiles ($Extend\...) are hidden by default. + _fs.NtfsOptions.HideMetafiles = false; + try + { + return _fs.GetFileId(path) & 0x0000FFFFFFFFFFFF; + } + catch (IOException) + { + return null; + } + finally + { + _fs.NtfsOptions.HideMetafiles = true; + } + } + + /// + /// DiscUtils returns paths with the platform's directory separator ('/' on Linux and macOS); the digests, + /// made on Windows, use '\'. + /// + private static string WindowsPath(string path) => path.Replace('/', '\\'); + + public ulong RecordLsn(long n) => BitConverter.ToUInt64(_mft, (int)(n * _recordSize) + 8); + + public string RecordHash(long n) => UsedHash(_mft, (int)(n * _recordSize), _recordSize, isIndex: false).Hash; + + /// + /// The duplicated information (times, sizes) in entries of is left out: + /// Windows refreshes it at mount without logging when it writes into those files. + /// + public IEnumerable<(string Directory, int Index, string Hash, ulong Lsn)> IndexBuffers(ISet excludedRecords) + { + var directories = new[] { @"\" }.Concat(_fs.GetDirectories(@"\", "*", SearchOption.AllDirectories)).ToList(); + foreach (var directory in directories.Where(d => !d.Contains("System Volume Information"))) + { + var name = (directory == @"\" ? @"\" : directory) + ":$I30:$INDEX_ALLOCATION"; + if (!_fs.FileExists(name)) + { + continue; + } + + var bytes = ReadAll(name); + for (var i = 0; (i + 1) * _indexSize <= bytes.Length; i++) + { + var (hash, lsn) = UsedHash(bytes, i * _indexSize, _indexSize, isIndex: true, excludedRecords); + yield return (WindowsPath(directory), i, hash, lsn); + } + } + } + + public string MftBitmapHash(ISet excludedRecords) => MaskedHash(ReadAll("$MFT::$BITMAP"), excludedRecords); + + public string ClusterBitmapHash(ISet excludedClusters) => MaskedHash(ReadAll("$Bitmap"), excludedClusters); + + public Dictionary ClusterOwners() + { + var owners = new Dictionary(); + for (long n = 0; n < RecordCount; n++) + { + var r = Logical(_mft, (int)(n * _recordSize), _recordSize); + if (BitConverter.ToUInt32(r, 0) != 0x454C4946 || (BitConverter.ToUInt16(r, 0x16) & 1) == 0) + { + continue; + } + + int a = BitConverter.ToUInt16(r, 0x14); + while (a + 8 < r.Length && BitConverter.ToUInt32(r, a) != 0xFFFFFFFF) + { + var length = (int)BitConverter.ToUInt32(r, a + 4); + if (length < 0x18 || a + length > r.Length) + { + break; + } + + if (r[a + 8] != 0) + { + var p = a + BitConverter.ToUInt16(r, a + 0x20); + long lcn = 0; + while (p < a + length && r[p] != 0) + { + var lengthSize = r[p] & 15; + var offsetSize = r[p] >> 4; + long runLength = 0; + for (var i = lengthSize - 1; i >= 0; i--) + { + runLength = (runLength << 8) | r[p + 1 + i]; + } + + if (offsetSize > 0) + { + long delta = 0; + for (var i = offsetSize - 1; i >= 0; i--) + { + delta = (delta << 8) | r[p + 1 + lengthSize + i]; + } + + if (offsetSize < 8 && (r[p + lengthSize + offsetSize] & 0x80) != 0) + { + delta |= -1L << (offsetSize * 8); + } + + lcn += delta; + for (long k = 0; k < runLength; k++) + { + owners[lcn + k] = n; + } + } + + p += 1 + lengthSize + offsetSize; + } + } + + a += length; + } + } + + return owners; + } + + /// Path, size, attributes, times, content hash and alternate streams of every file and directory. + public List Listing() + { + var lines = new List(); + foreach (var entry in _fs.GetFileSystemEntries(@"\", "*", SearchOption.AllDirectories).ToList().OrderBy(e => e, StringComparer.Ordinal)) + { + if (entry.Contains("System Volume Information")) + { + continue; + } + + var info = _fs.GetFileSystemInfo(entry); + var isDirectory = _fs.DirectoryExists(entry); + var line = $"{WindowsPath(entry)}|{(isDirectory ? "dir" : _fs.GetFileLength(entry).ToString())}|{(int)info.Attributes:x}|{info.CreationTimeUtc.Ticks}|{info.LastWriteTimeUtc.Ticks}"; + if (!isDirectory) + { + line += "|" + ContentHash(entry); + foreach (var stream in _fs.GetAlternateDataStreams(entry).OrderBy(s => s, StringComparer.Ordinal)) + { + line += $"|:{stream}={ContentHash(entry + ":" + stream)}"; + } + } + + lines.Add(line.Replace(' ', '_')); + } + + return lines; + } + + private string ContentHash(string path) + { + try + { + var bytes = ReadAll(path); + return bytes.Length + "/" + Hash(bytes, 0, bytes.Length); + } + catch (NotImplementedException) + { + // DiscUtils cannot read every kind of file (e.g. compressed and sparse at once). + return "unreadable"; + } + } + + private byte[] ReadAll(string path) + { + using var s = _fs.OpenFile(path, FileMode.Open, FileAccess.Read); + var ms = new MemoryStream(); + s.CopyTo(ms); + return ms.ToArray(); + } + + private static int Size(sbyte raw, int cluster) => raw < 0 ? 1 << -raw : raw * cluster; + + private static (string Hash, ulong Lsn) UsedHash(byte[] all, int offset, int length, bool isIndex, ISet? excludedRecords = null) + { + var b = Logical(all, offset, length); + var signature = BitConverter.ToUInt32(b, 0); + if (signature != (isIndex ? 0x58444E49u : 0x454C4946u)) + { + return (Hash(b, 0, b.Length), 0); + } + + var lsn = BitConverter.ToUInt64(b, 8); + var used = isIndex ? 0x18 + (int)BitConverter.ToUInt32(b, 0x1C) : (int)BitConverter.ToUInt32(b, 0x18); + var usaOffset = BitConverter.ToUInt16(b, 4); + var usaCount = BitConverter.ToUInt16(b, 6); + Array.Clear(b, 8, 8); + Array.Clear(b, usaOffset, 2 * usaCount); + if (isIndex && excludedRecords != null) + { + var entry = 0x18 + (int)BitConverter.ToUInt32(b, 0x18); + while (entry + 0x10 <= used && entry + 0x10 <= length) + { + var entryLength = BitConverter.ToUInt16(b, entry + 8); + if (entryLength < 0x10) + { + break; + } + + var record = (long)(BitConverter.ToUInt64(b, entry) & 0xFFFFFFFFFFFF); + if ((BitConverter.ToUInt16(b, entry + 0x0C) & 2) == 0 && excludedRecords.Contains(record) && entry + 0x50 <= length) + { + Array.Clear(b, entry + 0x18, 0x38); + } + + entry += entryLength; + } + } + + return (Hash(b, 0, Math.Min(Math.Max(used, 0), length)), lsn); + } + + private static byte[] Logical(byte[] all, int offset, int length) + { + var b = new byte[length]; + Array.Copy(all, offset, b, 0, length); + var usaOffset = BitConverter.ToUInt16(b, 4); + var usaCount = BitConverter.ToUInt16(b, 6); + if (usaCount >= 2 && usaOffset + (2 * usaCount) <= length && length % (usaCount - 1) == 0) + { + var stride = length / (usaCount - 1); + for (var i = 1; i < usaCount; i++) + { + b[(i * stride) - 2] = b[usaOffset + (2 * i)]; + b[(i * stride) - 1] = b[usaOffset + (2 * i) + 1]; + } + } + + return b; + } + + private static string MaskedHash(byte[] bitmap, ISet excludedBits) + { + var copy = (byte[])bitmap.Clone(); + foreach (var bit in excludedBits) + { + if (bit >> 3 < copy.Length) + { + copy[bit >> 3] &= (byte)~(1 << (int)(bit & 7)); + } + } + + return Hash(copy, 0, copy.Length); + } + } +} diff --git a/docs/ntfs-journal-hibernation.md b/docs/ntfs-journal-hibernation.md new file mode 100644 index 000000000..6d7db9832 --- /dev/null +++ b/docs/ntfs-journal-hibernation.md @@ -0,0 +1,54 @@ +# NTFS journal recovery and Windows hibernation + +## What a volume stream can show + +Windows writes the resume image to `hiberfil.sys` on the Windows volume. Full hibernation saves the system and user sessions. Fast Startup logs off users and saves the kernel session and drivers. A reduced hibernation file supports Fast Startup. Windows documents distinguishing the two resume modes through power transition information supplied to drivers. This information is unavailable to a reader of one NTFS volume. The guard makes the same decision for either kind of local resume image. [Microsoft system power states](https://learn.microsoft.com/en-us/windows/win32/power/system-power-states) · [Fast Startup and resume](https://learn.microsoft.com/en-us/windows-hardware/drivers/kernel/distinguishing-fast-startup-from-wake-from-hibernation) + +| Local observation | Recovery decision | Evidence | +| --- | --- | --- | +| The first four bytes of a readable header are `hibr` or `HIBR` | Positive local evidence of a saved resume image. Refuse. | NTFS-3G checks both spellings. Windows hibernation file studies describe these signatures in the `PO_MEMORY_IMAGE` header. [NTFS-3G hiberfile check](https://github.com/tuxera/ntfs-3g/blob/7f0f841fc52cf719106c5c93bafe465004e36816/libntfs-3g/volume.c#L786-L850) · [Sylve et al., 2016](https://doi.org/10.1016/j.diin.2016.12.003) | +| The header starts with `HORM` | Positive local evidence. Refuse. | The format study identifies the signature. Microsoft documents Hibernate Once/Resume Many reusing the saved resume image. [Format study](https://www.cct.lsu.edu/~golden/Papers/sylvehiber.pdf) · [Microsoft HORM documentation](https://learn.microsoft.com/en-us/windows/configuration/unified-write-filter/hibernate-once-resume-many-horm) | +| The 4 KiB header is physically readable and zeroed, or starts with `wake` or `WAKE` | No local evidence of an active resume image. | The libhibr format notes describe `wake` and `WAKE` as invalidated resume images. A Windows 8–10 study observed `WAKE` after resume and found it could persist through full shutdown. In the project maintainer's forensic experience across Windows versions, Windows can retain `hiberfil.sys` after resume while clearing its former contents with TRIM, leaving the first 4 KiB entirely zero. Such zeros indicate no local saved resume image, not that the volume is necessarily safe to modify. [Windows measurements](https://www.cct.lsu.edu/~golden/Papers/sylvehiber.pdf) · [libhibr format notes](https://github.com/libyal/libhibr/blob/main/documentation/Windows%20Hibernation%20File%20%28hiberfil.sys%29%20format.asciidoc#212-signature) | +| `RSTR` or `rstr`, other nonzero header bytes, a file shorter than 4 KiB, or a failed lookup or read | Indeterminate. Refuse. | The format study identifies `RSTR` as a transition during resume. NTFS-3G refuses short files and read errors. Its short-file diagnostic refers to a non-system partition. DiscUtils treats a short file as ambiguous because truncation can produce the same result. [Format study](https://www.cct.lsu.edu/~golden/Papers/sylvehiber.pdf) · [NTFS-3G hiberfile check](https://github.com/tuxera/ntfs-3g/blob/7f0f841fc52cf719106c5c93bafe465004e36816/libntfs-3g/volume.c#L796-L850) | +| A readable root index contains no `hiberfil.sys` | No local evidence. Ordinary NTFS data volumes remain eligible for recovery. | The resume image lives on the Windows volume. An NTFS data volume can lack a local file whether or not Windows still has state for that volume. [Microsoft system power states](https://learn.microsoft.com/en-us/windows/win32/power/system-power-states) · [NTFS-3G mount guidance](https://github.com/tuxera/ntfs-3g/blob/7f0f841fc52cf719106c5c93bafe465004e36816/src/ntfs-3g.8.in) | + +NTFS-3G 2026.9.28 checks the root file before read-write mount. It also separately refuses LFS 2.0 because Windows may have cached metadata, and the implementation explicitly observes that an ordinary abrupt termination can leave the same version. The five DiscUtils Windows crash images use LFS 2.0. The version remains a format selector in DiscUtils. The volume dirty bit can indicate outstanding writes or corruption. The boot geometry and journal restart state describe the volume and its log. None of these fields establishes Windows' power state. [NTFS-3G mount checks](https://github.com/tuxera/ntfs-3g/blob/7f0f841fc52cf719106c5c93bafe465004e36816/libntfs-3g/volume.c#L678-L729) · [Microsoft dirty-bit documentation](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/fsutil-dirty) · [Windows crash-image tests](../Tests/LibraryTests/Ntfs/Journal/NtfsJournalWindowsSampleTest.cs) + +The project maintainer has also observed the documented header signatures consistently in forensic applications on recent Windows versions. This practical experience supports the signature classification without requiring fresh Windows 11 captures as a merge prerequisite. It does not establish universal ownership safety or prove support for every possible file layout. Unknown nonzero headers and unreadable mappings remain indeterminate and prevent recovery. + +Linux NTFS3 has a separate read-only replay path and tracks whether the journal still needs replay. Its dirty-volume remount check and `force` option provide no basis for treating a stream as free of saved Windows state. The older Linux `ntfs` implementation is a separate driver. Its local check recognizes a completely zeroed 4 KiB header and treats unfamiliar nonzero data conservatively. [Linux NTFS3 replay](https://code.googlesource.com/linux/torvalds/linux/+/4bf54e47525dcacd4c6cdd97fb5902592414dd7a/fs/ntfs3/fslog.c#4844) · [NTFS3 remount policy](https://github.com/torvalds/linux/blob/master/fs/ntfs3/super.c) · [Linux ntfs local check](https://code.googlesource.com/linux/torvalds/linux/+/68ff4a3ccda9f98c74f23c70c8c7c581f9eee931/fs/ntfs/super.c#1054) + +## Recovery admission + +The probe reads only the supplied NTFS volume stream. It uses the root `$I30` lookup with `$UpCase`, follows the matching file reference, and reads the first 4 KiB of the unnamed `$DATA` from the mapped clusters. The prefix must be initialized and physically mapped in the first attribute extent. Synthesized sparse or uninitialized zeros, special layouts, ambiguous references and lookup or read failures prevent admission. A recognized signature is a reason to refuse or a marker of local invalidation. The probe does not validate the version-specific memory image or its checksum. + +The guard runs for otherwise admissible journal recovery, including reset when there are no changed metadata pages. It checks both the original volume and, when there are staged changes, a private read-only view of the proposed result. The original view preserves evidence that redo could remove. The staged view can expose a file or header that redo would make visible. If a planned repair can replace a torn `$MFT` bootstrap record, the original lookup can bootstrap from the validated `$MFTMirr` location. The staged view must have a readable primary. The guard completes before any metadata, mirror or restart-page write and introduces no backing-stream flushes. + +| Entry point or mode | Behavior when the guard refuses | +| --- | --- | +| `NtfsJournal.Analyze` | Returns `Unsupported` with the reason and leaves the backing stream untouched. | +| `NtfsJournal.Replay` and writable `Replay` construction, including the factory | Throws `IOException` before writing. | +| Read-only `Replay`, including the read-only default | Reports `Unsupported` and omits the overlay. The original view is available if its metadata can be read. A replayed view cannot account for Windows' suspended memory state. | +| `Refuse`, including the writable default | Applies the existing refusal for an unsupported journal. | +| `Ignore` | Bypasses journal handling and the local probe. It never initiates journal recovery. | + +An empty or clean journal is a no-op and does not run the hibernation probe. `Clean`, `NoLocalEvidence` and `CanReplayPersistently` cannot establish exclusive ownership. A data volume may still be owned by Windows when the resume image is elsewhere. Callers must supply an offline volume with exclusive access and establish that no suspended or running operating system owns it. The guard cannot reconstruct missing evidence, attest the entire directory tree or detect an out-of-band change during recovery. + +## Windows qualification + +The added tests use synthetic headers, metadata damage, read faults and staged replay. They exercise the stream and admission paths. The five unchanged Windows crash images qualify the existing recovery comparisons on captured data volumes. They were produced by abrupt VM power-off and do not qualify hibernation detection. Published measurements, libhibr's format notes and the maintainer's continuing forensic observations support the header distinctions. Current Windows 11 captures would provide additional independent qualification of the file layout and the observed states, but are not a merge prerequisite. [Existing Windows capture procedure](ntfs-journal-recovery-qualification.md) + +1. Use a disposable Windows 11 VM with an NTFS Windows volume and a second disposable NTFS virtual disk. Use a hypervisor configuration where the guest reports the relevant options through `powercfg /a`. Record the Windows build, firmware, virtual disk format and sector sizes. Keep the second volume mounted in the guest. Use guest power transitions and keep the host's volumes outside the test. +2. Start each run from a clone of the same clean guest. Enable hibernation in the guest with `powercfg /hibernate on` and run each transition below separately. Repeat the full-shutdown case after a verified successful resume to capture a possible consumed header. A VM save-state or a rejected power command does not qualify the run. [Microsoft shutdown command reference](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/shutdown/) + + | Capture | Guest command | + | --- | --- | + | Full hibernation | `powercfg /h /type full`, then `shutdown.exe /h` | + | Fast Startup | `shutdown.exe /s /hybrid /t 0` with Fast Startup available | + | Full shutdown with hibernation enabled | `shutdown.exe /s /t 0` | + +3. After the guest powers off, copy both virtual disks before booting or mounting them. Hash the copies. Use a read-only image reader to expose each NTFS partition stream. Record the root entry, its unnamed data allocation and initialized lengths, the first 4 KiB signature, the `$Volume` flags and the journal version and state. Corroborate the header and allocation with an independent read-only reader. Keep the saved memory contents private. +4. In a local `LibraryTests` case, run `NtfsHibernation.Probe(volume, JournalVolumeLayout.Read(volume))` on each captured partition and record `NtfsJournal.Analyze(volume)`. Verify positive local evidence for a confirmed active system capture and verify the observed classification of the inactive capture. Record what is visible on the secondary disk independently of the primary. A missing local marker on the secondary disk is expected to be inconclusive. +5. Exercise persistent replay on a separate disposable copy only if the journal needs recovery. Check that refusal leaves the complete image hash identical. If the active Windows captures all have clean journals, use a separate synthetic recoverable journal on a derivative copy to exercise admission. Keep the native header observation and synthetic admission result distinct. + +As an optional follow-up, obtain current Windows evidence for full hibernation, Fast Startup and a full shutdown with hibernation enabled. Record any unexpected header, uninitialized prefix or layout refusal before deciding whether the policy needs to change. These synthetic tests and published observations do not constitute native Windows 11 file-layout qualification. The secondary-volume limitation remains even if the captures pass. diff --git a/docs/ntfs-journal-recovery-qualification.md b/docs/ntfs-journal-recovery-qualification.md new file mode 100644 index 000000000..9f399e126 --- /dev/null +++ b/docs/ntfs-journal-recovery-qualification.md @@ -0,0 +1,27 @@ +# NTFS journal undo and interrupted replay + +## What the evidence covers + +The Windows crash images from PR #87 compare DiscUtils recovery with Windows on the same captured volumes. The PR reports 13 comparisons and 25 hard power-offs, none of which produced an unfinished transaction whose undo changed metadata. The five images in the test suite remain useful for redo and mount comparisons. They do not qualify metadata-changing undo. [PR #87 and its validation procedure](https://github.com/LTRData/DiscUtils/pull/87) + +Windows Internals describes checkpoint analysis, forward redo and rollback of uncommitted transactions through backward links. It also describes logging recovery's undo operations so Windows can repeat them after another interruption. That account establishes the need for restartable undo. It does not establish the current on-disk compensation format, prepared transaction outcomes, ordering for overlapping transactions or a replacement for a compensation LSN. [Windows Internals, NTFS Recovery Support](https://flylib.com/books/en/4.491.1.110/1/) + +DiscUtils builds the transaction table from checkpoint entries and later records. A commit resolves the transaction, a forget removes its slot, and a logged compensation or end of a top-level action can advance the undo cursor. An unresolved prepare is refused. For active transactions the in-memory replay follows each validated backward chain. It refuses cross-client or cross-transaction pointers, links outside the transaction's retained history, missing or remapped undo targets, and pending rollback in two transactions that touch the same physical cluster. Bitmap bit runs are the exception: two transactions may roll back different bits of one cluster, because setting or clearing disjoint bits gives the same result in any order. They are refused only when their bits overlap. Those are limits of the implementation. They do not define what Windows can recover. + +The former `LastLsn + 1` marker had no corresponding log record. It also made the second inverse on the same cached page skip its work. In-memory undo now keeps the page's historical LSN and applies the chain without using a page stamp as an undo cursor. A page LSN outside the usable history for an inverse operation causes refusal. `CanReplayPersistently` is false when the journal needs metadata undo, including when the net overlay has no changes. Explicit persistent replay refuses before writes or journal reset. An inverse already recorded in the journal can still be redone as a logged action. + +## What the interruption tests model + +Tests use a live byte view and a separate persisted image. A completed barrier transfers pending writes into the persisted image. A cut can discard pending writes or retain the specified prefix of the failing write. Each retry starts from the persisted image. The tests exercise mirror and primary writes, metadata barriers, both restart-page writes, and a second cut during retry. A simulated torn primary that lacks a usable replacement is refused. A complete replacement can be redone even if the torn header already contains its LSN. + +Replay stages all writes, flushes mirror changes before the primary MFT, and flushes metadata before resetting either restart page. It publishes and flushes the alternate restart page before overwriting the dirty source. A recognizable clean reset paired with its dirty source is completed only if reanalysis agrees with the reset LSN and finds no pending metadata changes or undo. A torn second write can leave a single valid clean restart page and reduced redundancy. Equal-LSN copies that conflict about the need for recovery or the active checkpoint are refused unless they form a matching reset pair. + +This model does not establish crash atomicity. A real backend must honor the barriers and the tested sector-write assumptions. Recovery cannot detect every inconsistent image produced by a backend that persisted a clean reset ahead of metadata. See [persistent replay and storage durability](ntfs-journal-replay-durability.md) for the flush behavior of supported stream configurations. + +## Bounded Windows VM qualification + +1. Use a disposable Windows VM and a second reference VM or a clean snapshot of the same build. Attach a newly created, fixed-size VHDX as a secondary data disk. Record the VM identity, Windows build, disk identity, sector and cluster sizes, and workload source and seed. Restrict formatting, workloads and crash attempts to the verified disposable data disk and test VM. Keep the host volume and guest system disk out of the experiment. +2. Save a powered-off baseline copy of the initialized data VHDX. Use a repeatable metadata-heavy workload on that disk, such as concurrent directory renames, index growth, file deletion and sparse allocation. Start from the baseline for each attempt. Bound the initial campaign to 30 hard power-offs or two hours, whichever comes first. Power off only the identified test VM. With it stopped, copy and hash the data VHDX before Windows can mount it again. A guest debugger can narrow the cut window in a follow-up when a recovery boundary is verified for the exact Windows build. +3. Inspect each capture offline using a read-only parser or manual log decoding independent of DiscUtils. Retain a candidate only if it has a valid checkpoint, an active transaction with a reachable metadata undo operation, and evidence that Windows must change metadata to roll it back. An absent commit alone is insufficient. Preserve the original capture, both restart pages, relevant log records and the target metadata before recovery. Exclude candidates produced by editing the captured journal or by an orderly dismount. +4. Give Windows and DiscUtils separate copies of the qualifying capture. Let Windows recover on first mount in the reference VM with no workload. Save the recovered image before running any repair. Capture filesystem consistency diagnostics separately. Compare the DiscUtils in-memory result against the Windows image at the metadata level, including the affected MFT records, index entries, allocation bits and file references. Record differing LSNs, update sequence values and mount-generated metadata explicitly. Use the log and the before and after images to distinguish a rollback from ordinary redo. Compare at least two independent Windows recoveries of the same capture before making it a fixture. +5. Preserve hashes, workload timing, log evidence, affected ranges and the Windows comparison result with any accepted fixture. Seek at least one confirmed file-record or index undo and one confirmed allocation-bitmap undo before drawing broader conclusions. If the attempt budget expires without a qualifying capture, record that outcome and keep persistent metadata undo disabled. A later campaign for persistent undo would also need independently observed compensation records, Windows remount behavior after interruption and backend-specific durability tests. diff --git a/docs/ntfs-journal-replay-durability.md b/docs/ntfs-journal-replay-durability.md new file mode 100644 index 000000000..aacac5853 --- /dev/null +++ b/docs/ntfs-journal-replay-durability.md @@ -0,0 +1,44 @@ +# NTFS persistent replay and storage durability + +`NtfsJournal.Replay` and explicit persistent replay on filesystem open use the same write path. Replay is not atomic. An interrupted call can leave partially updated metadata, and an exception can occur after bytes become visible or reach storage. Use an offline volume with exclusive access. The caller must establish the guarantees of the supplied stream and its backing storage. `Replayed` reports writes, a journal reset and successful returns from the flush calls on that stream. + +## Write sequence + +| Phase | Ordering dependency | Retry after an interruption | +| --- | --- | --- | +| Analyze and prepare | Validate admission, stage the metadata, and prepare the required restart pages before writing. The existing undo, client-slot, mapping and hibernation checks apply. | An unsupported state leaves the volume unchanged. | +| Write and flush `$MFTMirr` if needed | The corresponding primary MFT record can advance its page LSN only after the mirror barrier. | An unchanged primary can still be redone after a partial mirror write or a failed mirror flush. | +| Write and flush primary metadata | Every metadata update must pass the barrier before publishing a clean restart page. Flush even when there are no new metadata writes to drain any bytes left buffered by a previous attempt. | A valid protected page can skip an already-applied operation using its LSN. A torn page must have a complete supported replacement or replay refuses it. Supported raw updates can be applied again in log order. | +| Write and flush the alternate restart page | Keep the dirty source available until the alternate reset passes the barrier. | A torn alternate can be discarded if the dirty source remains valid. If the two pages form a recognized partial reset, replay rechecks the log end LSN and requires no pending metadata changes or undo. The preceding unconditional flush also covers a visible but buffered alternate on a retry through the same stream. | +| Write and flush the dirty source restart page | The alternate has already passed its barrier. | A torn source can leave one valid clean restart page and reduced redundancy. A failed final flush can leave two clean pages visible on the live stream. That retry is a clean no-op, while reopening from storage may still find a partial reset. | + +The order is the same for either source restart-page offset. Once a write or flush throws, replay issues no later writes. A backend can still retain part or all of the failed operation, or propagate buffered writes after the call returns. + +The first four MFT records are mirrored on ordinary layouts. The Windows `s64k` fixture has a 64 KiB `$MFTMirr` with matching reserved records beyond the fourth record and differing bytes later in the cluster. Replay copies complete declared records in the reserved range that fits in the mirror and leaves the remaining bytes as stored. Microsoft's NTFS developer notes reserve the first 16 records for special files. NTFS-3G sizes the mirror from the cluster geometry and limits its match check to 16 records based on observed Windows behavior. This evidence establishes a supported prefix for replay. It leaves the mirroring policy of later records across Windows versions unqualified. [Microsoft MFT notes](https://learn.microsoft.com/en-us/windows/win32/devnotes/master-file-table), [NTFS-3G mirror geometry](https://github.com/tuxera/ntfs-3g/blob/edge/libntfs-3g/bootsect.c), [NTFS-3G match check](https://github.com/tuxera/ntfs-3g/blob/edge/libntfs-3g/volume.c) + +## What a flush establishes + +A successful `Write` means the stream accepted the bytes. A subsequent read can see those bytes while they still reside in a managed buffer, an overlay or an OS cache. A wrapper can forward its pending data and flush to the next layer. An OS file synchronization call requests a further boundary. Power-loss survival depends on the filesystem, storage server or hypervisor, controller, media and their cache and write behavior. + +| Supplied stream or configuration | Flush behavior in replay | Established behavior and remaining limits | +| --- | --- | --- | +| Direct `FileStream` | A non-owning `SparseStream.FromStream` adapter requests `Flush(true)` | Requests OS file synchronization. The runtime, filesystem and storage must honor the request and report errors. This call does not establish sector atomicity or protection from every device failure. | +| Partition ranges and fixed VHD content | `SubStream.Flush(true)` forwards to its parent, including nested sparse wrappers | A backing `FileStream` receives `Flush(true)`; custom sparse wrappers can implement their own virtual `Flush(bool)` contract. | +| `SparseStream.FromStream`, synchronization, cache and other forwarding wrappers | Their `Flush(bool)` implementations forward the request through the wrapped stream | A wrapped `FileStream` receives `Flush(true)` and a wrapped `SparseStream` receives its virtual overload. Unknown `Stream` implementations receive `Flush()`. | +| Dynamic or differencing VHD content | Writable `DynamicStream.Flush(true)` forwards to its image file stream | Requests a backing disk flush when supported, after payload and current allocation writes have been issued. Deferred footers and container allocation crash behavior require separate qualification. | +| Writable VHDX content | `ContentStream.Flush(true)` forwards through its sparse image backing stream | This reaches the backing stream on the writable path, subject to any further wrapping. NTFS replay gains no guarantee about container-level crash recovery from that call alone. | +| Memory, a live snapshot, or a custom seekable stream | Virtual `SparseStream.Flush(bool)` defaults to `Flush()` if not overridden | A `MemoryStream` provides visibility in memory. An active `SnapshotStream` keeps overlay writes separate even when it flushes its base. Other implementations require an independently established contract. | + +`NtfsJournal.Replay` continues to accept `Stream`; it creates a non-owning `SparseStream` adapter for the flush boundaries only. There is no journal-specific inspection of a nested `FileStream`. An implementation that overrides the parameterless `Flush()` but also derives from a built-in forwarding wrapper should override `Flush(bool)` if it must intercept the stronger request. + +Modern `FileStream.Flush()` dispatches with `flushToDisk: false`. On Windows the disk-flush path uses `FlushFileBuffers`, including on Microsoft .NET Framework. On Unix, modern .NET requests file synchronization using the runtime's native implementation. Linux `fsync` waits for the device's reported completion. macOS distinguishes `fsync` from `F_FULLFSYNC`, which requests that the device flush its cache. The exact Unix primitive and error handling depend on the runtime version and platform. A .NET Standard build inherits the behavior of its host runtime. The .NET Framework-targeted tests run on Mono locally and check API dispatch, with native Windows coverage from CI. [FileStream API](https://learn.microsoft.com/en-us/dotnet/api/system.io.filestream.flush), [modern FileStream source](https://source.dot.net/System.Private.CoreLib/src/runtime/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs.html), [.NET Framework source](https://github.com/microsoft/referencesource/blob/main/mscorlib/system/io/filestream.cs), [OS synchronization APIs](https://learn.microsoft.com/en-us/dotnet/api/system.io.randomaccess.flushtodisk), [Linux fsync](https://man7.org/linux/man-pages/man2/fsync.2.html), [Apple fsync](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html) + +There is also a runtime error-reporting limitation. With .NET 10.0.0 on Linux, a probe against a disposable file observed `fsync` for `Flush(true)` and no `fsync` for `Flush()`. A replay probe on a copy of the `s4kn` fixture behind two built-in `SubStream` ranges reached `fsync` at all three reset boundaries. Injecting `EIO` into those calls still let replay return `Replayed = true`. This reproduces the upstream Unix `SystemNative_FSync` issue and tests error reporting. Qualify the deployed runtime and filesystem when relying on error propagation. [Runtime issue and syscall-injection reproduction](https://github.com/dotnet/runtime/issues/135201), [upstream correction](https://github.com/dotnet/runtime/pull/124725) + +## Evidence and remaining work + +The interruption tests track a live byte image separately from a simulated stable image. They check the issued operations and exact stable bytes at each boundary, including partial writes, a flush that throws before or after advancing the stable image, and reopening after the final successful flush. They cover same-stream retry, retry from the simulated stable image, and repeat recovery on a clean journal. Separate tests verify disk-flush dispatch for file and partition streams, propagation through sparse wrappers and dynamic VHD content, and fallback to parameterless flush for unknown streams. The five Windows crash-image comparisons exercise NTFS replay results. The extra large-cluster mirror update is synthetic and uses the geometry of one of those images. + +The model can also acknowledge a flush while discarding one metadata write. A recognizable partial reset with missing metadata is refused without further mutation. If both clean pages survive while that update is lost, reopening reports a clean journal and cannot diagnose the lost update. The tests establish these behaviors for controlled writes and barriers. Actual power loss can also damage previously flushed data or both restart pages, or interrupt a container allocation update. Those cases require qualification on the deployed storage stack. + +Keep the stream API and existing admission policy for this change. Before relying on power-loss durability for wrapped or remote storage, qualify the runtime, image container, filesystem and device using disposable images. The new overload requests a stronger flush where available but is not an end-to-end guarantee: custom wrappers and external storage layers must implement and honor the request. If an explicit contract is needed, a replay-specific option can accept a caller-provided barrier that covers the whole storage stack after each stream flush, propagates failure, and is invoked before any dependent write.