NTFS Journal support and improved SparseStream flush mechanism

Library/DiscUtils.Iscsi/DiscUtils.Iscsi.csproj

@@ -9,4 +9,8 @@
     <ProjectReference Include="..\DiscUtils.Core\DiscUtils.Core.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="LibraryTests" />
+  </ItemGroup>
+
 </Project>

Library/DiscUtils.Iscsi/Disk.cs

@@ -164,4 +164,29 @@ public override VirtualDisk CreateDifferencingDisk(string path, bool useAsync =
     {
         throw new NotSupportedException("Differencing disks not supported for iSCSI disks");
     }
-}
+
+    /// <summary>
+    /// Disposes of this instance. Like an operating system that unmounts a disk, it first asks the target to put
+    /// the writes it acknowledged on stable storage.
+    /// </summary>
+    /// <param name="disposing"><c>true</c> if running inside Dispose().</param>
+    /// <remarks>
+    /// A failure of that last flush is not reported here (the session may already be gone). Call
+    /// <c>Content.Flush()</c> before disposing to find out whether the target stored the writes.
+    /// </remarks>
+    protected override void Dispose(bool disposing)
+    {
+        if (disposing)
+        {
+            try
+            {
+                _stream?.Flush(flushToDisk: true);
+            }
+            catch (Exception e) when (e is IOException or ObjectDisposedException)
+            {
+            }
+        }
+
+        base.Dispose(disposing);
+    }
+}

Library/DiscUtils.Iscsi/DiskStream.cs

@@ -71,7 +71,24 @@ public override long Position
         set => _position = value;
     }
 
-    public override void Flush() { }
+    /// <summary>
+    /// Requests that the target synchronize its write cache with stable storage (SCSI SYNCHRONIZE CACHE).
+    /// This covers the entire LUN, including writes made through other streams or before this stream was opened.
+    /// A failed or unsupported synchronization is reported to the caller.
+    /// </summary>
+    public override void Flush() => Flush(flushToDisk: true);
+
+    /// <summary>
+    /// Requests a target-wide cache synchronization. iSCSI has no separate buffered data in this stream to flush,
+    /// so both overloads preserve the established <see cref="Flush()"/> behavior and issue SYNCHRONIZE CACHE.
+    /// In particular, <paramref name="flushToDisk"/> set to <c>true</c> is an unconditional durability barrier,
+    /// even if this stream has not written any data itself.
+    /// </summary>
+    public override void Flush(bool flushToDisk) => _session.SynchronizeCache(_lun);
+
+    /// <summary>Asynchronously synchronizes the whole LUN, even if this stream has not written anything.</summary>
+    public override async Task FlushAsync(CancellationToken cancellationToken)
+        => await _session.SynchronizeCacheAsync(_lun, cancellationToken).ConfigureAwait(false);
 
     public override int Read(byte[] buffer, int offset, int count)
         => Read(buffer.AsSpan(offset, count));
@@ -242,4 +259,4 @@ public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, Cancella
             }
         }
     }
-}
+}

Library/DiscUtils.Iscsi/README.md

@@ -22,6 +22,12 @@ dotnet add package LTRData.DiscUtils.Iscsi
 
 Initiator and Session provide explicit connections and LUN access. The package also supplies an iscsi transport for generic disk opening. This is a client library, not an iSCSI server or an operating-system initiator driver. Registration is AOT-compatible; protocol-key reflection elsewhere in iSCSI is outside that guarantee.
 
+### Write-cache synchronization
+
+`Disk.Content.Flush()` and `Disk.Content.Flush(true)` issue SCSI `SYNCHRONIZE CACHE (10)` for the entire LUN, with `IMMED` clear. The request is sent even when the current stream has not written anything: a stream-local dirty flag cannot establish whether the target has outstanding cached writes from an earlier session or another stream. `FlushAsync()` requests the same synchronization.
+
+An unsupported command (including SCSI `ILLEGAL REQUEST` / ASC `0x20`) or other synchronization failure is reported to the caller. An unsupported command does **not** establish that the target has no volatile write cache. The `Disk` disposal flush remains best-effort and suppresses errors; applications that need a durability barrier must call and check `Content.Flush(true)` explicitly before disposal. Even a successful command depends on the target and backing storage honoring SCSI cache synchronization.
+
 ## Registration
 
 For automatic discovery, register the providers implemented by this package:

Library/DiscUtils.Iscsi/ScsiSynchronizeCacheCommand.cs

@@ -0,0 +1,31 @@
+using System;
+
+namespace DiscUtils.Iscsi;
+
+/// <summary>
+/// SYNCHRONIZE CACHE (10): asks the target to put every write it has acknowledged on stable storage.
+/// </summary>
+/// <remarks>
+/// LBA 0 with 0 blocks covers the whole medium, also beyond what a 32-bit LBA can address. IMMED is clear, so the
+/// target only answers once the data is stored.
+/// </remarks>
+internal class ScsiSynchronizeCache10Command : ScsiCommand
+{
+    public ScsiSynchronizeCache10Command(ulong targetLun)
+        : base(targetLun) { }
+
+    public override int Size => 10;
+
+    public override TaskAttributes TaskAttributes => TaskAttributes.Simple;
+
+    public override int ReadFrom(ReadOnlySpan<byte> buffer)
+    {
+        throw new NotImplementedException();
+    }
+
+    public override void WriteTo(Span<byte> buffer)
+    {
+        buffer[..10].Clear();
+        buffer[0] = (byte)ScsiOpCode.SynchronizeCache10;
+    }
+}

Library/DiscUtils.Iscsi/Session.cs

@@ -387,6 +387,25 @@ public int RawCommand(long lun, ReadOnlyMemory<byte> command, ReadOnlySpan<byte>
         return Send(cmd, outBuffer, inBuffer);
     }
 
+    /// <summary>
+    /// Asks the target to put every write to a LUN that it has acknowledged on stable storage (SCSI SYNCHRONIZE CACHE).
+    /// </summary>
+    /// <param name="lun">The LUN to flush.</param>
+    internal void SynchronizeCache(long lun)
+    {
+        Send(new ScsiSynchronizeCache10Command((ulong)lun), default, default);
+    }
+
+    /// <summary>
+    /// Asks the target to put every write to a LUN that it has acknowledged on stable storage (SCSI SYNCHRONIZE CACHE).
+    /// </summary>
+    /// <param name="lun">The LUN to flush.</param>
+    /// <param name="cancellationToken"></param>
+    internal async ValueTask SynchronizeCacheAsync(long lun, CancellationToken cancellationToken)
+    {
+        await SendAsync(new ScsiSynchronizeCache10Command((ulong)lun), default, default, cancellationToken).ConfigureAwait(false);
+    }
+
     internal uint NextCommandSequenceNumber()
     {
         return ++CommandSequenceNumber;

Library/DiscUtils.Ntfs/Directory.cs

@@ -109,7 +109,7 @@ public override string ToString()
         return dir;
     }
 
-    internal DirectoryEntry? GetEntryByName(string name)
+    internal DirectoryEntry? GetEntryByName(string name, bool requireUnique = false)
     {
         var searchName = name;
 
@@ -119,13 +119,22 @@ public override string ToString()
             searchName = name[..streamSepPos];
         }
 
-        var entry = Index.FindFirst(new FileNameQuery(searchName, _context.UpperCase));
-        if (entry.Key != null)
+        using var matches = Index.FindAll(new FileNameQuery(searchName, _context.UpperCase), validate: requireUnique).GetEnumerator();
+        if (!matches.MoveNext())
         {
-            return new DirectoryEntry(this, entry.Value, entry.Key);
+            return null;
         }
 
-        return null;
+        var entry = matches.Current;
+        while (requireUnique && matches.MoveNext())
+        {
+            if (matches.Current.Value != entry.Value)
+            {
+                throw new InvalidDataException($"The directory has ambiguous entries for {name}");
+            }
+        }
+
+        return new DirectoryEntry(this, entry.Value, entry.Key);
     }
 
     internal DirectoryEntry AddEntry(File file, string name, FileNameNamespace nameNamespace)

Library/DiscUtils.Ntfs/FileSystemFactory.cs

@@ -20,6 +20,7 @@
 // DEALINGS IN THE SOFTWARE.
 //
 
+using System;
 using System.Collections.Generic;
 using System.IO;
 using DiscUtils.Vfs;
@@ -42,6 +43,19 @@ public override IEnumerable<FileSystemInfo> Detect(Stream stream, VolumeInfo vol
 
     private NtfsFileSystem Open(Stream stream, VolumeInfo volumeInfo, FileSystemParameters parameters)
     {
-        return new NtfsFileSystem(stream);
+        const string key = "NtfsJournalMode";
+        if (parameters == null || !parameters.Parameters.ContainsKey(key))
+        {
+            return new NtfsFileSystem(stream);
+        }
+
+        var text = parameters.Parameters[key]?.Trim();
+        if (!Enum.TryParse<NtfsJournalMode>(text, ignoreCase: true, out var mode) ||
+            !string.Equals(text, Enum.GetName(typeof(NtfsJournalMode), mode), StringComparison.OrdinalIgnoreCase))
+        {
+            throw new ArgumentException($"{key} must be Replay, Refuse or Ignore", nameof(parameters));
+        }
+
+        return new NtfsFileSystem(stream, mode);
     }
-}
+}

Library/DiscUtils.Ntfs/Index.cs

@@ -199,9 +199,16 @@ public static void Create(AttributeType attrType, AttributeCollationRule collati
         idx.WriteRootNodeToDisk();
     }
 
-    public IEnumerable<KeyValuePair<byte[], byte[]>> FindAll(IComparable<byte[]> query)
+    public IEnumerable<KeyValuePair<byte[], byte[]>> FindAll(IComparable<byte[]> query, bool validate = false)
     {
-        foreach (var entry in FindAllIn(query, _rootNode))
+        if (validate && (!IsFileIndex || _root.AttributeType != (uint)AttributeType.FileName ||
+            _root.CollationRule != AttributeCollationRule.Filename || _root.IndexAllocationSize != _bpb.IndexBufferSize ||
+            _root.RawClustersPerIndexRecord == 0))
+        {
+            throw new InvalidDataException("The directory index root has an invalid format");
+        }
+
+        foreach (var entry in FindAllIn(query, _rootNode, validate, depth: 0))
         {
             yield return new KeyValuePair<byte[], byte[]>(entry.KeyBuffer, entry.DataBuffer);
         }
@@ -414,8 +421,13 @@ protected IEnumerable<IndexEntry> Enumerate(IndexNode node)
         }
     }
 
-    private IEnumerable<IndexEntry> FindAllIn(IComparable<byte[]> query, IndexNode node)
+    private IEnumerable<IndexEntry> FindAllIn(IComparable<byte[]> query, IndexNode node, bool validate, int depth)
     {
+        if (validate)
+        {
+            ValidateLookupNode(node, depth);
+        }
+
         foreach (var focus in node.Entries)
         {
             var searchChildren = true;
@@ -441,8 +453,13 @@ private IEnumerable<IndexEntry> FindAllIn(IComparable<byte[]> query, IndexNode n
 
             if (searchChildren && (focus.Flags & IndexEntryFlags.Node) != 0)
             {
+                if (validate)
+                {
+                    ValidateLookupChild(focus.ChildrenVirtualCluster);
+                }
+
                 var block = GetSubBlock(focus);
-                foreach (var entry in FindAllIn(query, block.Node))
+                foreach (var entry in FindAllIn(query, block.Node, validate, depth + 1))
                 {
                     yield return entry;
                 }
@@ -460,6 +477,58 @@ private IEnumerable<IndexEntry> FindAllIn(IComparable<byte[]> query, IndexNode n
         }
     }
 
+    private void ValidateLookupNode(IndexNode node, int depth)
+    {
+        var header = node.Header;
+        if (depth >= 64 || header.OffsetToFirstEntry < IndexHeader.Size ||
+            header.OffsetToFirstEntry + (long)node.CalcEntriesSize() != header.TotalSizeOfEntries ||
+            header.TotalSizeOfEntries > header.AllocatedSizeOfEntries)
+        {
+            throw new InvalidDataException("The directory index has invalid node bounds or depth");
+        }
+
+        byte[] previousKey = null;
+        foreach (var entry in node.Entries)
+        {
+            if ((entry.Flags & ~(IndexEntryFlags.Node | IndexEntryFlags.End)) != 0)
+            {
+                throw new InvalidDataException("The directory index has an invalid entry flag");
+            }
+
+            if ((entry.Flags & IndexEntryFlags.End) != 0)
+            {
+                return;
+            }
+
+            var key = entry.KeyBuffer;
+            if (key.Length < 0x42 || key.Length != 0x42 + 2 * key[0x40] ||
+                (previousKey != null && _comparer.Compare(previousKey, key) > 0))
+            {
+                throw new InvalidDataException("The directory index has an invalid or out-of-order filename");
+            }
+
+            previousKey = key;
+        }
+
+        throw new InvalidDataException("The directory index node has no end entry");
+    }
+
+    private void ValidateLookupChild(long vcn)
+    {
+        if (vcn < 0 || vcn > long.MaxValue / Math.Max(_bpb.BytesPerCluster, (long)IndexBufferSize) ||
+            AllocationStream == null || _indexBitmap == null)
+        {
+            throw new InvalidDataException("The directory index has an invalid child reference");
+        }
+
+        var position = IndexBlockVcnToPosition(vcn);
+        if (position > AllocationStream.Length - IndexBufferSize || position % IndexBufferSize != 0 ||
+            !_indexBitmap.IsPresent(position / IndexBufferSize))
+        {
+            throw new InvalidDataException("The directory index child is outside the allocated index blocks");
+        }
+    }
+
     private void WriteRootNodeToDisk()
     {
         _rootNode.Header.AllocatedSizeOfEntries = (uint)_rootNode.CalcSize();

Library/DiscUtils.Ntfs/IndexView.cs

@@ -57,9 +57,9 @@ public D this[K key]
         set => _index[Unconvert(key)] = Unconvert(value);
     }
 
-    public IEnumerable<KeyValuePair<K, D>> FindAll(IComparable<byte[]> query)
+    public IEnumerable<KeyValuePair<K, D>> FindAll(IComparable<byte[]> query, bool validate = false)
     {
-        foreach (var entry in _index.FindAll(query))
+        foreach (var entry in _index.FindAll(query, validate))
         {
             yield return new KeyValuePair<K, D>(Convert<K>(entry.Key), Convert<D>(entry.Value));
         }