From f4504dfe9fbd17674c39d36dd1e98b4ddcc21214 Mon Sep 17 00:00:00 2001 From: Largo Date: Fri, 23 Oct 2026 15:55:02 +0200 Subject: [PATCH 1/8] Store into a boxed struct's field, not into a copy of it RubyFieldInfo built a field store as Assign(Field(Convert(target, S), f), v). Convert unboxes into a temporary, so once the rule is compiled the store changes that copy and the box Ruby holds keeps its old fields - silently. OpenSSL::PKey fills RSAParameters / DSAParameters / ECParameters this way, so a PEM loaded about 35 times and then failed for the rest of the process ("The specified RSA parameters are not valid"): the interpreter updated the box, compiled code did not. With today's upstream DLR the interpreter copies too (IronLanguages/dlr#368), and it fails on the first load. The store now goes through Unbox, which addresses the value inside the box, in compiled and interpreted code alike. Calls to mutating methods on a boxed struct already reached the box. ClrValueTypes1, commented out since 2011 as "fails with /noadaptive", is the same bug and runs again; ClrValueTypes3 fills S2 and RSAParameters in a method called 100 times and checks every result. --- Src/IronRuby.Tests/RubyTests.cs | 4 +-- Src/IronRuby.Tests/Runtime/ClrTests.cs | 41 +++++++++++++++++++++++++ Src/Ruby/Runtime/Calls/RubyFieldInfo.cs | 9 ++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/Src/IronRuby.Tests/RubyTests.cs b/Src/IronRuby.Tests/RubyTests.cs index 3937efc1b..a57f106f0 100644 --- a/Src/IronRuby.Tests/RubyTests.cs +++ b/Src/IronRuby.Tests/RubyTests.cs @@ -554,9 +554,9 @@ public Tests(Driver/*!*/ driver) { ClrToString1, ClrHashEquals4, ClrTypeVariance1, - // TODO: fails with /noadaptive - // ClrValueTypes1, + ClrValueTypes1, ClrValueTypes2, + ClrValueTypes3, HostingDefaultOptions1, Interactive1, Interactive2, diff --git a/Src/IronRuby.Tests/Runtime/ClrTests.cs b/Src/IronRuby.Tests/Runtime/ClrTests.cs index a9baed17d..722c39add 100644 --- a/Src/IronRuby.Tests/Runtime/ClrTests.cs +++ b/Src/IronRuby.Tests/Runtime/ClrTests.cs @@ -2419,6 +2419,47 @@ public void ClrValueTypes1() { Assert(unboxed.Y == 4); } + // A struct built and filled in Ruby (OpenSSL::PKey fills RSAParameters this way). The stores used to + // go to a temporary copy once the call sites were compiled, so after ~32 calls every field read nil. + public void ClrValueTypes3() { + Context.ObjectClass.SetConstant("S2", Context.GetClass(typeof(S2))); + Context.ObjectClass.SetConstant("RSAParameters", Context.GetClass(typeof(System.Security.Cryptography.RSAParameters))); + + AssertOutput(() => CompilerTest(@" +def fill(i) + s = S2.new + s.X = i + s.Y = i + 1 + s +end + +def set_x(i) + s = S2.new(i, i) + s.SetX + s +end + +def rsa(i) + p = RSAParameters.new + p.Modulus = System::Array[System::Byte].new(i + 1) + p +end + +bad = (0...100).reject { |i| s = fill(i); s.X == i && s.Y == i + 1 } +puts ""fill: #{bad.inspect}"" + +bad = (0...100).reject { |i| set_x(i).X == 1 } +puts ""set_x: #{bad.inspect}"" + +bad = (0...100).reject { |i| m = rsa(i).Modulus; m && m.Length == i + 1 } +puts ""rsa: #{bad.inspect}"" +"), @" +fill: [] +set_x: [] +rsa: [] +"); + } + public class ClassWithValueTypeField1 { public S2 Value = new S2(1, 2); } diff --git a/Src/Ruby/Runtime/Calls/RubyFieldInfo.cs b/Src/Ruby/Runtime/Calls/RubyFieldInfo.cs index a282f8372..51b3ce4e1 100644 --- a/Src/Ruby/Runtime/Calls/RubyFieldInfo.cs +++ b/Src/Ruby/Runtime/Calls/RubyFieldInfo.cs @@ -71,6 +71,15 @@ internal override void BuildCallNoFlow(MetaObjectBuilder/*!*/ metaBuilder, CallA Expression instance = _fieldInfo.IsStatic ? null : Ast.Convert(args.TargetExpression, _fieldInfo.DeclaringType); if (_isSetter) { + if (instance != null && _fieldInfo.DeclaringType.IsValueType) { + // Convert unboxes a struct into a temporary, so the store would change the copy and leave the + // box Ruby holds untouched - silently, and only once the rule is compiled (the interpreter used + // to update the box). Unbox addresses the value inside the box instead. + instance = args.TargetExpression.Type.IsValueType + ? args.TargetExpression + : Ast.Unbox(AstUtils.Convert(args.TargetExpression, typeof(object)), _fieldInfo.DeclaringType); + } + var actualArgs = RubyOverloadResolver.NormalizeArguments(metaBuilder, args, 1, 1); if (!metaBuilder.Error) { metaBuilder.Result = Ast.Assign( From ec2ce505d9d7ed442985e7f7e1b92b1e1d7cca7c Mon Sep 17 00:00:00 2001 From: Largo Date: Fri, 23 Oct 2026 23:07:46 +0200 Subject: [PATCH 2/8] Time: tests for the Windows machine zone and an Olson TZ Time_MachineZone1 compares Time.at, Time.local and Time.now with what TimeZoneInfo.Local says for a winter and a summer instant, so it holds in any machine zone, and checks that a local time's zone survives Marshal. Time_OlsonZone1 sets TZ=Europe/Zurich, which Windows resolves through TimeZoneInfo since it has no zoneinfo files. Both fail on 4.0.1 on Windows, where the zone was TimeZoneInfo.Local's base offset alone. --- Src/IronRuby.Tests/RubyTests.cs | 2 + Src/IronRuby.Tests/Runtime/MiscTests.cs | 53 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/Src/IronRuby.Tests/RubyTests.cs b/Src/IronRuby.Tests/RubyTests.cs index a57f106f0..3a8b1f589 100644 --- a/Src/IronRuby.Tests/RubyTests.cs +++ b/Src/IronRuby.Tests/RubyTests.cs @@ -700,6 +700,8 @@ public Tests(Driver/*!*/ driver) { Scenario_MainSingleton, Time_LoadZone1, Time_DumpLoadedZone1, + Time_MachineZone1, + Time_OlsonZone1, Scenario_RubyThreads1, Scenario_YieldCodeGen, diff --git a/Src/IronRuby.Tests/Runtime/MiscTests.cs b/Src/IronRuby.Tests/Runtime/MiscTests.cs index 3ba74ef5a..b08c920c8 100644 --- a/Src/IronRuby.Tests/Runtime/MiscTests.cs +++ b/Src/IronRuby.Tests/Runtime/MiscTests.cs @@ -813,6 +813,59 @@ puts ArrayList }, @"System::Collections::ArrayList"); } + /// + /// With TZ unset the zone is the machine's, daylight saving time included. Windows has no /etc/localtime, + /// and the zone used to be TimeZoneInfo.Local's base offset alone, so Time.now, Time.local and File.mtime + /// were an hour off all summer. Compared with .NET's own answer, so it holds in any machine zone. The zone's + /// name, which on Windows is in the installation's language and need not be ASCII, survives Marshal. + /// + public void Time_MachineZone1() { + TestOutput(@" +ENV.delete('TZ') +local = System::TimeZoneInfo.Local +# 2026-01-15 and 2026-07-15, 10:00 UTC +[1768471200, 1784109600].each do |s| + t = Time.at(s) + expected = local.GetUtcOffset(System::DateTimeOffset.FromUnixTimeSeconds(s)).TotalSeconds.to_i + dst = local.IsDaylightSavingTime(System::DateTimeOffset.FromUnixTimeSeconds(s)) + w = Time.local(t.year, t.month, t.day, t.hour, t.min, t.sec) + p [t.utc_offset == expected, t.isdst == dst, w.to_i == s, w.utc_offset == expected] +end +p Time.now.utc_offset == local.GetUtcOffset(System::DateTimeOffset.UtcNow).TotalSeconds.to_i + +t = Time.at(1784109600) +p Marshal.load(Marshal.dump(t)).zone == t.zone +", @" +[true, true, true, true] +[true, true, true, true] +true +true +"); + } + + /// + /// An Olson TZ name resolves on Windows too, where there is no zoneinfo file to read it from. + /// + public void Time_OlsonZone1() { + TestOutput(@" +saved = ENV['TZ'] +begin + ENV['TZ'] = 'Europe/Zurich' + w = Time.at(1768471200) # 2026-01-15 10:00 UTC + s = Time.at(1784109600) # 2026-07-15 10:00 UTC + p [w.hour, w.utc_offset, w.isdst], [s.hour, s.utc_offset, s.isdst] + p Time.local(2026, 7, 15, 12).to_i, Time.local(2026, 1, 15, 11).to_i +ensure + ENV['TZ'] = saved +end +", @" +[11, 3600, false] +[12, 7200, true] +1784109600 +1768471200 +"); + } + /// /// Time._load hands a dumped zone name to the class's find_timezone. As in MRI, a StandardError from the /// lookup leaves the name and anything else goes through; only bytes that are not a time are a TypeError. From 371e2464b20a5d9aa985d256b421bc1438f9e17c Mon Sep 17 00:00:00 2001 From: Largo Date: Fri, 23 Oct 2026 16:11:09 +0200 Subject: [PATCH 3/8] Windows: zlib on the zlib .NET ships require "zlib" was a LoadError on Windows: there is no libz there, and the binding's z_stream is laid out for LP64. Every .NET runtime carries zlib in System.IO.Compression.Native, though (zlib-ng since .NET 9, Intel's zlib in .NET 8), behind CompressionNative_* exports over its own PAL_ZStream. ClrZlib runs LibZ's entry points on those exports, so ZStream, Inflate, Deflate and the gzip classes in zlib.rb work unchanged: the z_stream record is only managed code's on Windows; next_in/avail_in/next_out/avail_out go to the PAL stream and back, and total_in, total_out and adler (the running adler32/crc32, or the dictionary id on Z_NEED_DICT) are kept as libz keeps them. adler32, crc32, their _combine and crc_table are computed as zlib does. A reset begins the stream again with its parameters (.NET 8 exports no reset). What the exports leave out raises NotImplementedError: set_dictionary, Deflate#params once data has gone in, Inflate#sync_point?. Linux and macOS keep calling libz exactly as before. ruby/spec library/zlib on Windows: 148 examples, 2 failures (they compare classic zlib's exact compressed bytes), 3 errors (the NotImplementedErrors); before, all 42 files were the LoadError. Zlib_Streams1 inflates input CRuby compressed and checks round trips, totals, adler and the checksums, so it holds on libz and on .NET's zlib alike. --- README.md | 17 +- Src/IronRuby.Tests/RubyTests.cs | 1 + Src/IronRuby.Tests/Runtime/MiscTests.cs | 113 +++++++ Src/Libraries/Zlib/ClrZlib.cs | 433 ++++++++++++++++++++++++ Src/Libraries/Zlib/zlib.cs | 140 ++++++-- Src/StdLib/ironruby/zlib.rb | 10 +- 6 files changed, 675 insertions(+), 39 deletions(-) create mode 100644 Src/Libraries/Zlib/ClrZlib.cs diff --git a/README.md b/README.md index a99342c55..31d7082ad 100644 --- a/README.md +++ b/README.md @@ -417,7 +417,7 @@ Linux is the primary target and the one the numbers above are measured on. **Win `ir.cmd` and `irb.cmd` run, and `spec/language` (2920 examples) and `spec/security` pass with **0 failures**. `spec/core` and `spec/library` are rougher there — symlinks/chmod/umask in `core/file`, fork and signals in `core/process`, and `IO.select`/`IO#wait` on pipes (which is -`poll(2)`, so `core/io` and the socket suites hang rather than fail). `zlib`, Win32OLE and +`poll(2)`, so `core/io` and the socket suites hang rather than fail). Win32OLE and readline are not implemented on Windows; `IO.console` is nil, so irb falls back to its ANSI input path. Windows Forms and WPF (`require "System.Windows.Forms"`, `"PresentationFramework"`) come from the Windows Desktop runtime, which `ir` references whenever it is built for Windows @@ -535,12 +535,17 @@ Where the Windows errors are: `core/file` (113) and `core/filetest` (8) are syml `chmod`/`umask` and the other Unix file metadata; `core/process` (36 failures, 15 errors) is `fork`, process groups, uid/gid and signals; `library/win32ole` (114) and `library/readline` (25) are libraries IronRuby does not implement and whose specs only run on Windows; -`library/zlib` (42) is the `LoadError` below. `core/objectspace` needs `-X:ObjectSpace`, as -on Linux. +`library/zlib` (2 failures, 3 errors) is the zlib below. `core/objectspace` needs +`-X:ObjectSpace`, as on Linux. -What Windows does not have yet: **`zlib`** (`require "zlib"` is a clean `LoadError` — there -is no libz to bind to, and the `z_stream` binding is laid out for an LP64 C compiler, so a -stray `zlib1.dll` would be read through the wrong struct), **`io/console`** (`IO.console` is +**`zlib`** on Windows runs on the zlib inside .NET itself (`System.IO.Compression.Native`: +zlib-ng from .NET 9 on, Intel's zlib in .NET 8), since there is no libz to bind to. It +inflates and deflates, gzip included, but its compressed bytes are that compressor's, not +classic zlib's (the 2 `library/zlib` failures compare exact bytes), and what .NET does not +export raises `NotImplementedError`: `set_dictionary`, `Deflate#params` once data has gone in, +and `Inflate#sync_point?` (the 3 errors). + +What Windows does not have yet: **`io/console`** (`IO.console` is `nil`, the termios binding being Unix-only by construction; irb and reline fall back to their ANSI path and start fine — `Src/Libraries/Termios` would need a Win32 console-API twin), **`IO#wait`/`IO.select` on a pipe** (they are `poll(2)`, which is why `io-wait`, diff --git a/Src/IronRuby.Tests/RubyTests.cs b/Src/IronRuby.Tests/RubyTests.cs index 3a8b1f589..6565fd130 100644 --- a/Src/IronRuby.Tests/RubyTests.cs +++ b/Src/IronRuby.Tests/RubyTests.cs @@ -702,6 +702,7 @@ public Tests(Driver/*!*/ driver) { Time_DumpLoadedZone1, Time_MachineZone1, Time_OlsonZone1, + Zlib_Streams1, Scenario_RubyThreads1, Scenario_YieldCodeGen, diff --git a/Src/IronRuby.Tests/Runtime/MiscTests.cs b/Src/IronRuby.Tests/Runtime/MiscTests.cs index b08c920c8..c06479cde 100644 --- a/Src/IronRuby.Tests/Runtime/MiscTests.cs +++ b/Src/IronRuby.Tests/Runtime/MiscTests.cs @@ -866,6 +866,119 @@ p Time.local(2026, 7, 15, 12).to_i, Time.local(2026, 1, 15, 11).to_i "); } + /// + /// Zlib's streams and checksums. Windows has no libz; there they run on the zlib .NET ships + /// (ClrZlib), so everything here holds on both: input CRuby compressed, round trips, and + /// the values libz keeps in the z_stream (total_in, total_out, adler) rather than bytes a + /// particular compressor produces. + /// + public void Zlib_Streams1() { + TestOutput(@" +load_assembly 'IronRuby.Libraries', 'IronRuby.StandardLibrary.Zlib' + +hex = lambda { |s| [s].pack('H*') } +text = 'hello hello hello hello' + +# from CRuby: Zlib::Deflate.deflate(text), and Zlib.gzip(text) +p Zlib::Inflate.inflate(hex['789ccb48cdc9c957c8402701680308b1']) +p Zlib.gunzip(hex['1f8b0800486adb6a0003cb48cdc9c957c8402701e3513d8d17000000']) + +# fed a few bytes at a time, as a PNG decoder does +data = 'IronRuby zlib ' * 2000 +z = Zlib::Deflate.deflate(data) +inf = Zlib::Inflate.new +out = '' +z.bytes.each_slice(7) { |s| out << inf.inflate(s.pack('C*')) } +out << inf.finish +p [out == data, inf.finished?, inf.total_in == z.bytesize, inf.total_out, inf.adler == Zlib.adler32(data)] +inf.close + +d = Zlib::Deflate.new +d << data[0, 10000] +d << data[10000..-1] +z = d.finish +p [d.total_in, d.total_out == z.bytesize, d.adler == Zlib.adler32(data), Zlib::Inflate.inflate(z) == data] +d.close + +p Zlib.gunzip(Zlib.gzip(data)) == data +raw = Zlib::Deflate.new(Zlib::BEST_SPEED, -Zlib::MAX_WBITS).deflate(data, Zlib::FINISH) +p Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(raw) == data +p Zlib::Inflate.new(Zlib::MAX_WBITS + 32).inflate(Zlib.gzip(data)) == data + +# a stream reset is a fresh one +d = Zlib::Deflate.new +d.deflate('discarded', Zlib::SYNC_FLUSH) +d.reset +z = d.deflate(text, Zlib::FINISH) +p [Zlib::Inflate.inflate(z), d.total_in] + +# deflated with the preset dictionary 'hello': inflate asks for it by its adler32 +inf = Zlib::Inflate.new +begin + inf.inflate(hex['78bb062c0215cb00110ae5f9453929001a0b045d']) +rescue Zlib::NeedDict + p [:need_dict, inf.adler == Zlib.adler32('hello')] +end + +p [Zlib.crc32(text), Zlib.adler32(text), Zlib.crc32, Zlib.adler32, Zlib.crc32('lo', Zlib.crc32('hel'))] +p [Zlib.crc32_combine(Zlib.crc32('hel'), Zlib.crc32('lo'), 2) == Zlib.crc32('hello'), + Zlib.adler32_combine(Zlib.adler32('hel'), Zlib.adler32('lo'), 2) == Zlib.adler32('hello')] +p Zlib.crc_table.size, Zlib.crc_table[1], Zlib.crc_table[255] + +begin + Zlib::Inflate.inflate('not zlib') +rescue Zlib::DataError => e + p e.message +end + +# deflateParams refuses bad values and leaves the stream as it was +d = Zlib::Deflate.new +begin + d.params(99, 0) +rescue Zlib::StreamError + p :refused +end +p Zlib::Inflate.inflate(d.deflate(text, Zlib::FINISH)) == text + +# the parameters a stream already has are no change, even after data (the default level is 6) +d = Zlib::Deflate.new +d << text +d.params(6, Zlib::DEFAULT_STRATEGY) +p Zlib::Inflate.inflate(d.finish) == text + +# deflate's guess at what the data is +d = Zlib::Deflate.new +d.deflate(text, Zlib::FINISH) +p d.data_type == Zlib::TEXT + +# inflate detecting the wrapper keeps adler at 1 until it has read a whole gzip header +inf = Zlib::Inflate.new(Zlib::MAX_WBITS + 32) +inf.inflate(Zlib.gzip(text)[0, 5]) +p inf.adler +", @" +""hello hello hello hello"" +""hello hello hello hello"" +[true, true, true, 28000, true] +[28000, true, true, true] +true +true +true +[""hello hello hello hello"", 23] +[:need_dict, true] +[2369606115, 1745029297, 0, 1, 907060870] +[true, true] +256 +1996959894 +755167117 +""incorrect header check"" +:refused +true +true +true +1 +"); + } + /// /// Time._load hands a dumped zone name to the class's find_timezone. As in MRI, a StandardError from the /// lookup leaves the name and anything else goes through; only bytes that are not a time are a TypeError. diff --git a/Src/Libraries/Zlib/ClrZlib.cs b/Src/Libraries/Zlib/ClrZlib.cs new file mode 100644 index 000000000..66024423b --- /dev/null +++ b/Src/Libraries/Zlib/ClrZlib.cs @@ -0,0 +1,433 @@ +/* **************************************************************************** + * + * This source code is subject to terms and conditions of the Apache License, Version 2.0. A + * copy of the license can be found in the License.html file at the root of this distribution. + * + * ***************************************************************************/ + +using System; +using System.Runtime.InteropServices; + +namespace IronRuby.StandardLibrary.Zlib { + + /// + /// zlib on Windows. There is no libz there, but every .NET runtime carries zlib in + /// System.IO.Compression.Native, the one DeflateStream runs on (zlib-ng in zlib-compat mode + /// since .NET 9, Intel's fork of zlib 1.2 in .NET 8). What it exports is narrower than libz: + /// init, deflate/inflate and end over its own PAL_ZStream, plus crc32. + /// + /// This runs LibZ's z_stream-shaped record over those exports, so ZStream cannot tell the + /// difference: next_in/avail_in/next_out/avail_out go to the PAL stream and back, total_in, + /// total_out and adler (the running adler32 or crc32 libz keeps) are kept here. What the + /// exports leave out - the preset dictionaries, deflateParams once data has gone in, and + /// inflateSyncPoint - raises NotImplementedError. + /// + /// The record's opaque field, which libz would hand to zalloc/zfree, holds the GCHandle of + /// the stream's ; nothing native ever reads the record. + /// + internal static class ClrZlib { + private const string Lib = "System.IO.Compression.Native"; + + /// The zlib API version behind the exports; libz's would come from zlibVersion(). + internal static string/*!*/ Version { + get { return Environment.Version.Major >= 9 ? "1.3.1.zlib-ng" : "1.2.13"; } + } + + /// PAL_ZStream from System.IO.Compression.Native's pal_zlib.h. + [StructLayout(LayoutKind.Sequential)] + private struct PalZStream { + public IntPtr nextIn; + public IntPtr nextOut; + public IntPtr msg; + public IntPtr internalState; + public uint availIn; + public uint availOut; + } + + [DllImport(Lib, EntryPoint = "CompressionNative_DeflateInit2_", CallingConvention = CallingConvention.Cdecl)] + private static extern int DeflateInit2_(ref PalZStream stream, int level, int method, int windowBits, int memLevel, int strategy); + + [DllImport(Lib, EntryPoint = "CompressionNative_Deflate", CallingConvention = CallingConvention.Cdecl)] + private static extern int Deflate(ref PalZStream stream, int flush); + + [DllImport(Lib, EntryPoint = "CompressionNative_DeflateEnd", CallingConvention = CallingConvention.Cdecl)] + private static extern int DeflateEnd(ref PalZStream stream); + + [DllImport(Lib, EntryPoint = "CompressionNative_InflateInit2_", CallingConvention = CallingConvention.Cdecl)] + private static extern int InflateInit2_(ref PalZStream stream, int windowBits); + + [DllImport(Lib, EntryPoint = "CompressionNative_Inflate", CallingConvention = CallingConvention.Cdecl)] + private static extern int Inflate(ref PalZStream stream, int flush); + + [DllImport(Lib, EntryPoint = "CompressionNative_InflateEnd", CallingConvention = CallingConvention.Cdecl)] + private static extern int InflateEnd(ref PalZStream stream); + + private const int Z_BINARY = 0; + private const int Z_TEXT = 1; + private const int Z_UNKNOWN = 2; + private const int Z_DEFAULT_COMPRESSION = -1; + private const int Z_FIXED = 4; + private const int GzipHeaderSize = 10; + + /// Which checksum libz would keep in strm->adler for the stream's wrapper. + private enum Check { None, Adler32, Crc32, Detect } + + private sealed class Stream { + public PalZStream Pal; + public bool IsInflate; + public int Level, Method, WindowBits, MemLevel, Strategy; + public Check Check; + public uint Sum; + + // The first bytes of the stream: a zlib header's dictionary id, which Z_NEED_DICT reports, + // and the fixed part of a gzip header, until which libz keeps adler at 1 when it detects gzip. + public readonly byte[]/*!*/ Header = new byte[GzipHeaderSize]; + public int HeaderLength; + public bool GzipHeaderPending; + + // deflate's data_type: libz guesses it from the first block it flushes (detect_data_type) + public bool DataTypeDecided, SawBinary, SawText; + + public int Init() { + Pal = default(PalZStream); + Sum = 0; + HeaderLength = 0; + GzipHeaderPending = false; + DataTypeDecided = SawBinary = SawText = false; + if (IsInflate) { + // inflateReset: strm->adler = state->wrap & 1 + Check = (WindowBits < 0) ? Check.None : (WindowBits >= 32) ? Check.Detect : (WindowBits >= 16) ? Check.Crc32 : Check.Adler32; + Sum = (Check == Check.Adler32 || Check == Check.Detect) ? 1u : 0u; + return InflateInit2_(ref Pal, WindowBits); + } + // deflateReset: strm->adler = wrap == 2 ? crc32(0, NULL, 0) : adler32(0, NULL, 0) + Check = (WindowBits < 0) ? Check.None : (WindowBits >= 16) ? Check.Crc32 : Check.Adler32; + Sum = (Check == Check.Crc32) ? 0u : 1u; + return DeflateInit2_(ref Pal, Level, Method, WindowBits, MemLevel, Strategy); + } + + public int End() { + return IsInflate ? InflateEnd(ref Pal) : DeflateEnd(ref Pal); + } + } + + private static LibZ.ZStreamRec Read(IntPtr strm) { + return (LibZ.ZStreamRec)Marshal.PtrToStructure(strm, typeof(LibZ.ZStreamRec)); + } + + private static void Write(IntPtr strm, LibZ.ZStreamRec rec) { + Marshal.StructureToPtr(rec, strm, false); + } + + private static Stream GetStream(LibZ.ZStreamRec rec) { + return (rec.opaque != IntPtr.Zero) ? (Stream)GCHandle.FromIntPtr(rec.opaque).Target : null; + } + + internal static int DeflateInit(IntPtr strm, int level, int method, int windowBits, int memLevel, int strategy) { + return Init(strm, new Stream { Level = level, Method = method, WindowBits = windowBits, MemLevel = memLevel, Strategy = strategy }); + } + + internal static int InflateInit(IntPtr strm, int windowBits) { + return Init(strm, new Stream { IsInflate = true, WindowBits = windowBits }); + } + + private static int Init(IntPtr strm, Stream/*!*/ s) { + int err = s.Init(); + var rec = Read(strm); + rec.msg = s.Pal.msg; + if (err == LibZ.Z_OK) { + rec.opaque = GCHandle.ToIntPtr(GCHandle.Alloc(s)); + rec.adler = (UIntPtr)s.Sum; + rec.total_in = rec.total_out = UIntPtr.Zero; + rec.data_type = InitialDataType(s); + } else { + // The PAL allocated its z_stream before zlib refused the parameters, and only End + // frees it; nothing will call End for a stream that never got a handle. + s.End(); + } + Write(strm, rec); + return err; + } + + // deflateReset sets Z_UNKNOWN; inflate leaves the zeroed field alone until it runs + private static int InitialDataType(Stream/*!*/ s) { + return s.IsInflate ? Z_BINARY : Z_UNKNOWN; + } + + internal static int End(IntPtr strm) { + var rec = Read(strm); + Stream s = GetStream(rec); + if (s == null) { + return LibZ.Z_STREAM_ERROR; + } + int err = s.End(); + GCHandle.FromIntPtr(rec.opaque).Free(); + rec.opaque = IntPtr.Zero; + rec.msg = IntPtr.Zero; + Write(strm, rec); + return err; + } + + internal static int Reset(IntPtr strm) { + var rec = Read(strm); + Stream s = GetStream(rec); + if (s == null) { + return LibZ.Z_STREAM_ERROR; + } + // .NET 8 exports no inflateReset and no runtime exports deflateReset; a stream begun + // again with the same parameters is what a reset leaves. + s.End(); + int err = s.Init(); + rec.msg = s.Pal.msg; + rec.adler = (UIntPtr)s.Sum; + rec.total_in = rec.total_out = UIntPtr.Zero; + rec.data_type = InitialDataType(s); + Write(strm, rec); + return err; + } + + internal static int Run(IntPtr strm, int flush) { + var rec = Read(strm); + Stream s = GetStream(rec); + if (s == null) { + return LibZ.Z_STREAM_ERROR; + } + + IntPtr inStart = rec.next_in, outStart = rec.next_out; + uint availIn = rec.avail_in, availOut = rec.avail_out; + s.Pal.nextIn = inStart; + s.Pal.availIn = availIn; + s.Pal.nextOut = outStart; + s.Pal.availOut = availOut; + + int err = s.IsInflate ? Inflate(ref s.Pal, flush) : Deflate(ref s.Pal, flush); + + int consumed = (int)(availIn - s.Pal.availIn); + int produced = (int)(availOut - s.Pal.availOut); + rec.next_in = s.Pal.nextIn; + rec.avail_in = s.Pal.availIn; + rec.next_out = s.Pal.nextOut; + rec.avail_out = s.Pal.availOut; + rec.msg = s.Pal.msg; + rec.total_in = (UIntPtr)((ulong)rec.total_in + (ulong)consumed); + rec.total_out = (UIntPtr)((ulong)rec.total_out + (ulong)produced); + + if (s.IsInflate) { + if (consumed > 0 && s.HeaderLength < s.Header.Length) { + int n = Math.Min(consumed, s.Header.Length - s.HeaderLength); + Marshal.Copy(inStart, s.Header, s.HeaderLength, n); + s.HeaderLength += n; + } + if (s.Check == Check.Detect && s.HeaderLength > 0) { + // libz keeps adler at 1 (the zlib start) until it has read a whole gzip header + s.Check = (s.Header[0] == 0x1f) ? Check.Crc32 : Check.Adler32; + s.GzipHeaderPending = s.Check == Check.Crc32; + } + if (s.GzipHeaderPending && s.HeaderLength >= GzipHeaderSize) { + s.GzipHeaderPending = false; + s.Sum = 0; + } + // the checksum covers what comes out of inflate and what goes into deflate + Update(s, outStart, produced); + if (err == LibZ.Z_NEED_DICT && s.HeaderLength >= 6) { + // inflate reports the dictionary it wants in strm->adler (RFC 1950's DICTID) + s.Sum = (uint)(s.Header[2] << 24 | s.Header[3] << 16 | s.Header[4] << 8 | s.Header[5]); + } + } else { + Update(s, inStart, consumed); + if (!s.DataTypeDecided) { + ScanDataType(s, inStart, consumed); + // libz decides when it flushes its first block; a flush here is where that + // happens for all but very long input, and the guess covers everything so far + if (flush != LibZ.Z_NO_FLUSH) { + s.DataTypeDecided = true; + rec.data_type = (!s.SawBinary && s.SawText) ? Z_TEXT : Z_BINARY; + } + } + } + rec.adler = (UIntPtr)s.Sum; + + Write(strm, rec); + return err; + } + + private static void Update(Stream/*!*/ s, IntPtr data, int length) { + if (length <= 0 || (s.Check != Check.Adler32 && s.Check != Check.Crc32)) { + return; + } + var bytes = new byte[length]; + Marshal.Copy(data, bytes, 0, length); + s.Sum = (s.Check == Check.Adler32) ? Checksums.Adler32(s.Sum, bytes, 0, length) : Checksums.Crc32(s.Sum, bytes, 0, length); + } + + /// + /// zlib's detect_data_type: a byte from the block list (0-6, 14-25, 28-31) makes the data + /// binary, otherwise one from the allow list (tab, line feed, carriage return, 32-255) makes + /// it text. + /// + private static void ScanDataType(Stream/*!*/ s, IntPtr data, int length) { + if (length <= 0 || s.SawBinary) { + return; + } + const uint blockMask = 0xf3ffc07f; + var bytes = new byte[length]; + Marshal.Copy(data, bytes, 0, length); + foreach (byte b in bytes) { + if (b < 32 && (blockMask & (1u << b)) != 0) { + s.SawBinary = true; + return; + } + if (b >= 32 || b == 9 || b == 10 || b == 13) { + s.SawText = true; + } + } + } + + /// + /// deflateParams. Until anything has gone in the stream can simply begin again with the new + /// level and strategy, which is all deflateParams does then; after that the exports have + /// nothing that changes them. Bad values are refused before anything changes, as in libz. + /// + internal static int Params(IntPtr strm, int level, int strategy) { + var rec = Read(strm); + Stream s = GetStream(rec); + if (s == null) { + return LibZ.Z_STREAM_ERROR; + } + if (level < Z_DEFAULT_COMPRESSION || level > 9 || strategy < 0 || strategy > Z_FIXED) { + return LibZ.Z_STREAM_ERROR; + } + if (NormalLevel(level) == NormalLevel(s.Level) && strategy == s.Strategy) { + return LibZ.Z_OK; + } + if ((ulong)rec.total_in != 0 || (ulong)rec.total_out != 0) { + throw Unavailable("Zlib::Deflate#params after data has been deflated", "deflateParams"); + } + s.Level = level; + s.Strategy = strategy; + return Reset(strm); + } + + // Z_DEFAULT_COMPRESSION is level 6 + private static int NormalLevel(int level) { + return (level == Z_DEFAULT_COMPRESSION) ? 6 : level; + } + + internal static Exception/*!*/ Unavailable(string/*!*/ what, string/*!*/ function) { + return new NotImplementedException(what + " is not available on Windows: the zlib .NET ships there (System.IO.Compression.Native) does not export " + function); + } + } + + /// adler32 and crc32 as libz computes them, for the platform without libz. + internal static class Checksums { + private const uint AdlerBase = 65521; + private const uint CrcPolynomial = 0xedb88320; + + internal static readonly uint[]/*!*/ CrcTable = MakeCrcTable(); + + private static uint[]/*!*/ MakeCrcTable() { + var table = new uint[256]; + for (uint n = 0; n < 256; n++) { + uint c = n; + for (int k = 0; k < 8; k++) { + c = ((c & 1) != 0) ? CrcPolynomial ^ (c >> 1) : c >> 1; + } + table[n] = c; + } + return table; + } + + internal static uint Adler32(uint adler, byte[] buffer, int offset, int length) { + if (buffer == null) { + return 1; + } + uint a = adler & 0xffff, b = adler >> 16; + while (length > 0) { + // 5552 is the most bytes that cannot overflow b before the modulo (zlib's NMAX) + int n = Math.Min(length, 5552); + length -= n; + while (n-- > 0) { + a += buffer[offset++]; + b += a; + } + a %= AdlerBase; + b %= AdlerBase; + } + return b << 16 | a; + } + + internal static uint Crc32(uint crc, byte[] buffer, int offset, int length) { + if (buffer == null) { + return 0; + } + uint c = ~crc; + for (int i = 0; i < length; i++) { + c = CrcTable[(c ^ buffer[offset + i]) & 0xff] ^ (c >> 8); + } + return ~c; + } + + /// zlib's adler32_combine. + internal static uint Adler32Combine(uint adler1, uint adler2, long length2) { + if (length2 < 0) { + return 0xffffffff; + } + uint rem = (uint)(length2 % AdlerBase); + ulong sum1 = adler1 & 0xffff; + ulong sum2 = (rem * sum1) % AdlerBase; + sum1 += (adler2 & 0xffff) + AdlerBase - 1; + sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + AdlerBase - rem; + if (sum1 >= AdlerBase) sum1 -= AdlerBase; + if (sum1 >= AdlerBase) sum1 -= AdlerBase; + if (sum2 >= ((ulong)AdlerBase << 1)) sum2 -= ((ulong)AdlerBase << 1); + if (sum2 >= AdlerBase) sum2 -= AdlerBase; + return (uint)(sum1 | (sum2 << 16)); + } + + /// zlib's crc32_combine: crc1 shifted by length2 zero bytes, then crc2. + internal static uint Crc32Combine(uint crc1, uint crc2, long length2) { + return MultiplyModP(PowerOfXModP(length2, 3), crc1) ^ crc2; + } + + // a(x) * b(x) modulo the CRC polynomial, bits reflected as in zlib's multmodp + private static uint MultiplyModP(uint a, uint b) { + uint m = 1u << 31, p = 0; + while (m != 0) { + if ((a & m) != 0) { + p ^= b; + if ((a & (m - 1)) == 0) { + break; + } + } + m >>= 1; + b = ((b & 1) != 0) ? (b >> 1) ^ CrcPolynomial : b >> 1; + } + return p; + } + + private static readonly uint[]/*!*/ _x2n = MakeX2nTable(); + + private static uint[]/*!*/ MakeX2nTable() { + var table = new uint[32]; + uint p = 1u << 30; // x^1 + table[0] = p; + for (int n = 1; n < 32; n++) { + table[n] = p = MultiplyModP(p, p); + } + return table; + } + + // x^(n * 2^k) modulo the CRC polynomial, as zlib's x2nmodp + private static uint PowerOfXModP(long n, int k) { + uint p = 1u << 31; // x^0 + while (n > 0) { + if ((n & 1) != 0) { + p = MultiplyModP(_x2n[k & 31], p); + } + n >>= 1; + k++; + } + return p; + } + } +} diff --git a/Src/Libraries/Zlib/zlib.cs b/Src/Libraries/Zlib/zlib.cs index fb3f4ab9c..d7d59c4b2 100644 --- a/Src/Libraries/Zlib/zlib.cs +++ b/Src/Libraries/Zlib/zlib.cs @@ -30,10 +30,15 @@ namespace IronRuby.StandardLibrary.Zlib { /// /// Compressed output has to be byte-for-byte what CRuby produces - the specs compare exact /// bytes - so the compressor itself is the system's, not a reimplementation. + /// + /// Windows has no libz; there the same entry points run on the zlib inside .NET itself + /// (), and the z_stream record below is only ever read by managed code. /// internal static class LibZ { private const string Lib = "libz"; + internal static readonly bool UseClrZlib = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + internal const int Z_NO_FLUSH = 0; internal const int Z_SYNC_FLUSH = 2; internal const int Z_FINISH = 4; @@ -84,7 +89,7 @@ internal struct ZStreamRec { internal static string Version { get { if (_version == null) { - _version = Marshal.PtrToStringAnsi(sys_zlibVersion()) ?? "1.2.11"; + _version = UseClrZlib ? ClrZlib.Version : (Marshal.PtrToStringAnsi(sys_zlibVersion()) ?? "1.2.11"); } return _version; } @@ -92,57 +97,148 @@ internal static string Version { internal static readonly int StreamSize = Marshal.SizeOf(typeof(ZStreamRec)); + internal static int deflateInit2_(IntPtr strm, int level, int method, int windowBits, + int memLevel, int strategy, string/*!*/ version, int stream_size) { + return UseClrZlib + ? ClrZlib.DeflateInit(strm, level, method, windowBits, memLevel, strategy) + : sys_deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size); + } + + internal static int deflate(IntPtr strm, int flush) { + return UseClrZlib ? ClrZlib.Run(strm, flush) : sys_deflate(strm, flush); + } + + internal static int deflateEnd(IntPtr strm) { + return UseClrZlib ? ClrZlib.End(strm) : sys_deflateEnd(strm); + } + + internal static int deflateReset(IntPtr strm) { + return UseClrZlib ? ClrZlib.Reset(strm) : sys_deflateReset(strm); + } + + internal static int deflateParams(IntPtr strm, int level, int strategy) { + return UseClrZlib ? ClrZlib.Params(strm, level, strategy) : sys_deflateParams(strm, level, strategy); + } + + internal static int deflateSetDictionary(IntPtr strm, byte[]/*!*/ dictionary, uint dictLength) { + if (UseClrZlib) { + throw ClrZlib.Unavailable("Zlib::Deflate#set_dictionary", "deflateSetDictionary"); + } + return sys_deflateSetDictionary(strm, dictionary, dictLength); + } + + internal static int inflateInit2_(IntPtr strm, int windowBits, string/*!*/ version, int stream_size) { + return UseClrZlib ? ClrZlib.InflateInit(strm, windowBits) : sys_inflateInit2_(strm, windowBits, version, stream_size); + } + + internal static int inflate(IntPtr strm, int flush) { + return UseClrZlib ? ClrZlib.Run(strm, flush) : sys_inflate(strm, flush); + } + + internal static int inflateEnd(IntPtr strm) { + return UseClrZlib ? ClrZlib.End(strm) : sys_inflateEnd(strm); + } + + internal static int inflateReset(IntPtr strm) { + return UseClrZlib ? ClrZlib.Reset(strm) : sys_inflateReset(strm); + } + + internal static int inflateSetDictionary(IntPtr strm, byte[]/*!*/ dictionary, uint dictLength) { + if (UseClrZlib) { + throw ClrZlib.Unavailable("Zlib::Inflate#set_dictionary", "inflateSetDictionary"); + } + return sys_inflateSetDictionary(strm, dictionary, dictLength); + } + + internal static int inflateSyncPoint(IntPtr strm) { + if (UseClrZlib) { + throw ClrZlib.Unavailable("Zlib::Inflate#sync_point?", "inflateSyncPoint"); + } + return sys_inflateSyncPoint(strm); + } + + internal static UIntPtr adler32(UIntPtr adler, byte[] buf, uint len) { + return UseClrZlib ? (UIntPtr)Checksums.Adler32((uint)adler, buf, 0, (int)len) : sys_adler32(adler, buf, len); + } + + internal static UIntPtr crc32(UIntPtr crc, byte[] buf, uint len) { + return UseClrZlib ? (UIntPtr)Checksums.Crc32((uint)crc, buf, 0, (int)len) : sys_crc32(crc, buf, len); + } + + internal static UIntPtr adler32_combine(UIntPtr adler1, UIntPtr adler2, long len2) { + return UseClrZlib ? (UIntPtr)Checksums.Adler32Combine((uint)adler1, (uint)adler2, len2) : sys_adler32_combine(adler1, adler2, len2); + } + + internal static UIntPtr crc32_combine(UIntPtr crc1, UIntPtr crc2, long len2) { + return UseClrZlib ? (UIntPtr)Checksums.Crc32Combine((uint)crc1, (uint)crc2, len2) : sys_crc32_combine(crc1, crc2, len2); + } + + /// The 256 entries of zlib's CRC-32 table. + internal static uint[]/*!*/ GetCrcTable() { + if (UseClrZlib) { + return (uint[])Checksums.CrcTable.Clone(); + } + // get_crc_table() hands back 256 z_crc_t (uint32 since zlib 1.2.7) entries. + var entries = new int[256]; + Marshal.Copy(sys_get_crc_table(), entries, 0, 256); + var result = new uint[256]; + for (int i = 0; i < entries.Length; i++) { + result[i] = unchecked((uint)entries[i]); + } + return result; + } + [DllImport(Lib, EntryPoint = "deflateInit2_", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - internal static extern int deflateInit2_(IntPtr strm, int level, int method, int windowBits, + private static extern int sys_deflateInit2_(IntPtr strm, int level, int method, int windowBits, int memLevel, int strategy, string/*!*/ version, int stream_size); [DllImport(Lib, EntryPoint = "deflate", CallingConvention = CallingConvention.Cdecl)] - internal static extern int deflate(IntPtr strm, int flush); + private static extern int sys_deflate(IntPtr strm, int flush); [DllImport(Lib, EntryPoint = "deflateEnd", CallingConvention = CallingConvention.Cdecl)] - internal static extern int deflateEnd(IntPtr strm); + private static extern int sys_deflateEnd(IntPtr strm); [DllImport(Lib, EntryPoint = "deflateReset", CallingConvention = CallingConvention.Cdecl)] - internal static extern int deflateReset(IntPtr strm); + private static extern int sys_deflateReset(IntPtr strm); [DllImport(Lib, EntryPoint = "deflateParams", CallingConvention = CallingConvention.Cdecl)] - internal static extern int deflateParams(IntPtr strm, int level, int strategy); + private static extern int sys_deflateParams(IntPtr strm, int level, int strategy); [DllImport(Lib, EntryPoint = "deflateSetDictionary", CallingConvention = CallingConvention.Cdecl)] - internal static extern int deflateSetDictionary(IntPtr strm, byte[]/*!*/ dictionary, uint dictLength); + private static extern int sys_deflateSetDictionary(IntPtr strm, byte[]/*!*/ dictionary, uint dictLength); [DllImport(Lib, EntryPoint = "inflateInit2_", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - internal static extern int inflateInit2_(IntPtr strm, int windowBits, string/*!*/ version, int stream_size); + private static extern int sys_inflateInit2_(IntPtr strm, int windowBits, string/*!*/ version, int stream_size); [DllImport(Lib, EntryPoint = "inflate", CallingConvention = CallingConvention.Cdecl)] - internal static extern int inflate(IntPtr strm, int flush); + private static extern int sys_inflate(IntPtr strm, int flush); [DllImport(Lib, EntryPoint = "inflateEnd", CallingConvention = CallingConvention.Cdecl)] - internal static extern int inflateEnd(IntPtr strm); + private static extern int sys_inflateEnd(IntPtr strm); [DllImport(Lib, EntryPoint = "inflateReset", CallingConvention = CallingConvention.Cdecl)] - internal static extern int inflateReset(IntPtr strm); + private static extern int sys_inflateReset(IntPtr strm); [DllImport(Lib, EntryPoint = "inflateSetDictionary", CallingConvention = CallingConvention.Cdecl)] - internal static extern int inflateSetDictionary(IntPtr strm, byte[]/*!*/ dictionary, uint dictLength); + private static extern int sys_inflateSetDictionary(IntPtr strm, byte[]/*!*/ dictionary, uint dictLength); [DllImport(Lib, EntryPoint = "inflateSyncPoint", CallingConvention = CallingConvention.Cdecl)] - internal static extern int inflateSyncPoint(IntPtr strm); + private static extern int sys_inflateSyncPoint(IntPtr strm); [DllImport(Lib, EntryPoint = "adler32", CallingConvention = CallingConvention.Cdecl)] - internal static extern UIntPtr adler32(UIntPtr adler, byte[] buf, uint len); + private static extern UIntPtr sys_adler32(UIntPtr adler, byte[] buf, uint len); [DllImport(Lib, EntryPoint = "crc32", CallingConvention = CallingConvention.Cdecl)] - internal static extern UIntPtr crc32(UIntPtr crc, byte[] buf, uint len); + private static extern UIntPtr sys_crc32(UIntPtr crc, byte[] buf, uint len); [DllImport(Lib, EntryPoint = "adler32_combine", CallingConvention = CallingConvention.Cdecl)] - internal static extern UIntPtr adler32_combine(UIntPtr adler1, UIntPtr adler2, long len2); + private static extern UIntPtr sys_adler32_combine(UIntPtr adler1, UIntPtr adler2, long len2); [DllImport(Lib, EntryPoint = "crc32_combine", CallingConvention = CallingConvention.Cdecl)] - internal static extern UIntPtr crc32_combine(UIntPtr crc1, UIntPtr crc2, long len2); + private static extern UIntPtr sys_crc32_combine(UIntPtr crc1, UIntPtr crc2, long len2); [DllImport(Lib, EntryPoint = "get_crc_table", CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr get_crc_table(); + private static extern IntPtr sys_get_crc_table(); } [RubyModule("Zlib")] @@ -355,13 +451,9 @@ public static object AdlerCombine(ConversionStorage/*!*/ integerCo [RubyMethod("crc_table", RubyMethodAttributes.PublicSingleton)] [RubyMethod("crc_table", RubyMethodAttributes.PrivateInstance)] public static RubyArray/*!*/ CrcTable(object self) { - // get_crc_table() hands back 256 z_crc_t (uint32 since zlib 1.2.7) entries. - IntPtr table = LibZ.get_crc_table(); var result = new RubyArray(256); - var entries = new int[256]; - Marshal.Copy(table, entries, 0, 256); - for (int i = 0; i < entries.Length; i++) { - result.Add(Protocols.Normalize(unchecked((uint)entries[i]))); + foreach (uint entry in LibZ.GetCrcTable()) { + result.Add(Protocols.Normalize(entry)); } return result; } diff --git a/Src/StdLib/ironruby/zlib.rb b/Src/StdLib/ironruby/zlib.rb index 0d20114c4..0450f4935 100644 --- a/Src/StdLib/ironruby/zlib.rb +++ b/Src/StdLib/ironruby/zlib.rb @@ -13,18 +13,10 @@ # # **************************************************************************** -# There is no libz to bind to on Windows, and the z_stream this binds to is laid out for -# an LP64 C compiler - `unsigned long` is 8 bytes there and 4 under MSVC - so even a -# zlib1.dll that happens to be on PATH would be read through the wrong struct. Refusing -# here is a LoadError the caller can rescue; letting it through was a -# TypeInitializationException from the first constant the class defines. -if ::File::ALT_SEPARATOR == "\\" - raise LoadError, "cannot load such file -- zlib (no libz on this platform)" -end - load_assembly 'IronRuby.Libraries', 'IronRuby.StandardLibrary.Zlib' # Zlib::ZStream, Inflate and Deflate are the libz binding in Src/Libraries/Zlib/zlib.cs. +# Windows has no libz; there they run on the zlib .NET itself ships (Src/Libraries/Zlib/ClrZlib.cs). # The gzip family below is written on top of them: a gzip member is a 10-byte header, a # raw deflate stream and an 8-byte footer, none of which needs libz directly. From 037e82b3d48d68651795182df02977160febdff3 Mon Sep 17 00:00:00 2001 From: Largo Date: Fri, 23 Oct 2026 16:27:29 +0200 Subject: [PATCH 4/8] Windows: File::NULL is NUL, and Process.kill without libc Process.spawn(cmd, out: File::NULL) failed on Windows with ENOENT, and Process.kill raised System::DllNotFoundException. IO::NULL (and File::NULL) came from the prelude as "/dev/null" everywhere; Windows has no such file, so the redirect could not be opened. It is "NUL" there, as in CRuby. The error itself named the program, not the file: WindowsSpawn handed back only -errno and the prelude filled in the command. A redirection that cannot be opened now raises with its own path - ENOENT, EACCES, or EISDIR for a directory - as CRuby does, which opens it in the parent. Process.kill P/Invoked libc's kill(), which Windows does not have. WindowsSpawn.Kill does what CRuby's win32 kill() does: KILL is TerminateProcess (exit code 0, as there), 0 reports whether the process still runs (ESRCH once it has exited), INT is a console Ctrl+C - Ctrl+Break aimed at a process group - and every other signal is EINVAL. ruby/spec on Windows: core/process/kill 2 errors -> 0; core/process/spawn, core/kernel/spawn and core/kernel/system unchanged otherwise. Process_SpawnKill1 spawns a child that blocks on a pipe, signals 0 and KILL, waits, and checks a missing redirection is named. --- Src/IronRuby.Tests/RubyTests.cs | 1 + Src/IronRuby.Tests/Runtime/MiscTests.cs | 57 +++++++++++ Src/Libraries/Builtins/PosixSignals.cs | 13 +++ Src/Libraries/Builtins/WindowsSpawn.cs | 121 +++++++++++++++++++++++- Src/StdLib/ironruby/ruby4.rb | 5 +- 5 files changed, 190 insertions(+), 7 deletions(-) diff --git a/Src/IronRuby.Tests/RubyTests.cs b/Src/IronRuby.Tests/RubyTests.cs index 6565fd130..66c2fbeda 100644 --- a/Src/IronRuby.Tests/RubyTests.cs +++ b/Src/IronRuby.Tests/RubyTests.cs @@ -703,6 +703,7 @@ public Tests(Driver/*!*/ driver) { Time_MachineZone1, Time_OlsonZone1, Zlib_Streams1, + Process_SpawnKill1, Scenario_RubyThreads1, Scenario_YieldCodeGen, diff --git a/Src/IronRuby.Tests/Runtime/MiscTests.cs b/Src/IronRuby.Tests/Runtime/MiscTests.cs index c06479cde..930f73648 100644 --- a/Src/IronRuby.Tests/Runtime/MiscTests.cs +++ b/Src/IronRuby.Tests/Runtime/MiscTests.cs @@ -866,6 +866,63 @@ p Time.local(2026, 7, 15, 12).to_i, Time.local(2026, 1, 15, 11).to_i "); } + /// + /// Process.spawn redirected to File::NULL, and Process.kill. On Windows File::NULL was "/dev/null", + /// so the redirect failed with an ENOENT that named the program; Process.kill P/Invoked libc's + /// kill() and raised DllNotFoundException. The child reads a pipe nobody writes to, so it runs + /// until it is killed. File::NULL and Process.spawn are the prelude's (ruby4.rb), which needs prism. + /// + [Options(Prism = true)] + public void Process_SpawnKill1() { + TestOutput(@" +require 'ruby4' +windows = File::ALT_SEPARATOR == '\\' +p File::NULL == (windows ? 'NUL' : '/dev/null'), IO::NULL == File::NULL + +command = windows ? ['cmd.exe', '/c', 'more'] : ['cat'] +r, w = IO.pipe +pid = Process.spawn(*(command + [{:in => r, :out => File::NULL, :err => File::NULL}])) +r.close +p Process.kill(0, pid), Process.kill(:KILL, pid) +p Process.wait(pid) == pid +w.close +begin + Process.kill(0, pid) +rescue Errno::ESRCH + p :gone +end + +# a redirection that cannot be opened is what the error names (Windows; posix_spawn cannot tell) +begin + Process.spawn(*(command + [{:out => 'no/such/dir/out.txt'}])) +rescue Errno::ENOENT => e + p(!windows || e.message.include?('no/such/dir/out.txt')) +end + +# a signal the process sends itself from another thread runs its handler on the main thread +# (Windows, where there is no kill(2) to deliver it; elsewhere the kernel does) +if windows + got = nil + old = trap(:TERM) { got = :handled } + Thread.new { Process.kill(:TERM, Process.pid) }.join + 20.times { break if got; sleep 0.05 } + trap(:TERM, old) + p got +else + p :handled +end +", @" +true +true +1 +1 +true +:gone +true +:handled +"); + } + /// /// Zlib's streams and checksums. Windows has no libz; there they run on the zlib .NET ships /// (ClrZlib), so everything here holds on both: input CRuby compressed, round trips, and diff --git a/Src/Libraries/Builtins/PosixSignals.cs b/Src/Libraries/Builtins/PosixSignals.cs index be5f720c9..a9af0f93d 100644 --- a/Src/Libraries/Builtins/PosixSignals.cs +++ b/Src/Libraries/Builtins/PosixSignals.cs @@ -186,6 +186,18 @@ internal static bool HasHandler(int signal) { /// to the prelude, which knows how to turn it into the right class. /// internal static int Kill(int pid, int signal) { + if (RubyProcess.IsWindows) { + // MRI's win32 kill() raise()s a signal a process sends itself (any but 0 and KILL), so + // its trap handler runs. The main thread has run it already (RubyProcess.Kill); from any + // other thread it is parked for the main one, as a delivered signal would be. + if (pid == Environment.ProcessId && signal != 0 && signal != SignalKill) { + if (HasHandler(signal)) { + Enqueue(signal); + } + return 0; + } + return WindowsSpawn.Kill(pid, signal); + } if (SysKill(pid, signal) == 0) { return 0; } @@ -403,6 +415,7 @@ private static void Enqueue(int signal) { } private const int SignalInterrupt = 2; + private const int SignalKill = 9; /// /// MRI answers the canonical spelling of a symbolic handler, not the one it was given: diff --git a/Src/Libraries/Builtins/WindowsSpawn.cs b/Src/Libraries/Builtins/WindowsSpawn.cs index 925b22b3e..626ae0994 100644 --- a/Src/Libraries/Builtins/WindowsSpawn.cs +++ b/Src/Libraries/Builtins/WindowsSpawn.cs @@ -58,6 +58,9 @@ internal static class WindowsSpawn { private const int ERROR_FILE_NOT_FOUND = 2; private const int ERROR_PATH_NOT_FOUND = 3; private const int ERROR_ACCESS_DENIED = 5; + private const int ERROR_SHARING_VIOLATION = 32; + private const int ERROR_LOCK_VIOLATION = 33; + private const int ERROR_INVALID_NAME = 123; private const int ERROR_BAD_EXE_FORMAT = 193; private const int ERROR_BROKEN_PIPE = 109; @@ -143,6 +146,15 @@ private static extern bool DuplicateHandle(IntPtr sourceProcess, IntPtr source, private static extern IntPtr CreateFileW(string name, uint access, uint share, IntPtr security, uint creation, uint flags, IntPtr template); + [DllImport("kernel32", SetLastError = true)] + private static extern IntPtr OpenProcess(uint access, bool inheritHandle, int processId); + + [DllImport("kernel32", SetLastError = true)] + private static extern bool TerminateProcess(IntPtr handle, uint exitCode); + + [DllImport("kernel32", SetLastError = true)] + private static extern bool GenerateConsoleCtrlEvent(uint ctrlEvent, int processGroupId); + #endregion #region children @@ -438,10 +450,12 @@ private static int ToInt(object value) { break; default: { - IntPtr opened = OpenFile(ToStr(action[2]), ToInt(action[3])); + string path = ToStr(action[2]); + IntPtr opened = OpenFile(path, ToInt(action[3])); if (opened == new IntPtr(-1)) { - plan.Error = -Errno(Marshal.GetLastWin32Error()); - return plan; + int error = Marshal.GetLastWin32Error(); + plan.Dispose(); + throw RedirectionError(error, path); } plan.Opened.Add(opened); Assign(plan, target, opened); @@ -504,11 +518,28 @@ private static IntPtr OpenFile(string/*!*/ path, int flags) { FILE_ATTRIBUTE_NORMAL, IntPtr.Zero); } + /// + /// MRI opens a redirection's file in the parent before it starts the child, so a file that + /// cannot be opened is what the error names. Handing back only -errno made the prelude name + /// the program instead: `out: "C:/no/dir/x"` read as "No such file or directory - ruby.exe". + /// + private static Exception/*!*/ RedirectionError(int win32Error, string/*!*/ path) { + switch (Errno(win32Error)) { + case ENOENT: return new FileNotFoundException(path); + case EACCES: return Directory.Exists(path) ? RubyExceptions.CreateEISDIR(path) : new UnauthorizedAccessException(path); + default: return new InvalidError(path); + } + } + + // the nearest errno, as MRI's map_errno gives it private static int Errno(int win32Error) { switch (win32Error) { case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: return ENOENT; - case ERROR_ACCESS_DENIED: return EACCES; + case ERROR_PATH_NOT_FOUND: + case ERROR_INVALID_NAME: return ENOENT; + case ERROR_ACCESS_DENIED: + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: return EACCES; case ERROR_BAD_EXE_FORMAT: return ENOEXEC; default: return EINVAL; } @@ -598,6 +629,86 @@ internal static int Exec(RubyContext/*!*/ context, MutableString/*!*/ file, Ruby #endregion + #region kill + + private const int SIGINT = 2; + private const int SIGKILL = 9; + private const int EPERM = 1; + private const int ESRCH = 3; + private const uint STILL_ACTIVE = 259; + private const uint PROCESS_TERMINATE = 0x0001; + private const uint PROCESS_QUERY_LIMITED_INFORMATION = 0x1000; + private const int ERROR_INVALID_PARAMETER = 87; + private const uint CTRL_C_EVENT = 0; + private const uint CTRL_BREAK_EVENT = 1; + + /// + /// kill(2) as MRI's win32 kill() does it, since there is no kill(2) to call - the libc one + /// Process.kill used to P/Invoke is not there (DllNotFoundException). Returns 0, or -errno. + /// KILL is TerminateProcess (with exit code 0, as in MRI), 0 asks whether the process still + /// runs, INT is a console Ctrl+C - Ctrl+Break for a pid, which has to be a process group + /// (spawn's pgroup: true). Windows has no other signals to send: EINVAL, as in MRI. + /// + internal static int Kill(int pid, int signal) { + if (pid < 0 || (pid == 0 && signal != SIGINT)) { + return -EINVAL; + } + + switch (signal) { + case SIGINT: + // a console's Ctrl+C cannot be aimed at one process group, so a pid gets Ctrl+Break + if (!GenerateConsoleCtrlEvent(pid != 0 ? CTRL_BREAK_EVENT : CTRL_C_EVENT, pid)) { + int error = Marshal.GetLastWin32Error(); + return -(error == 0 ? EPERM : Errno(error)); + } + return 0; + + case 0: + case SIGKILL: + // A child's handle is the one __waitpid__ reaps and closes, so it is used with the + // table locked: otherwise another thread could close it between the lookup and the + // call, and a child started meanwhile could be handed the same handle value. + lock (_children) { + IntPtr child; + if (_children.TryGetValue(pid, out child)) { + return Signal(child, signal); + } + } + + // signal 0 only asks, so it needs no right to end the process + uint access = PROCESS_QUERY_LIMITED_INFORMATION | (signal == SIGKILL ? PROCESS_TERMINATE : 0); + IntPtr handle = OpenProcess(access, false, pid); + if (handle == IntPtr.Zero) { + return (Marshal.GetLastWin32Error() == ERROR_INVALID_PARAMETER) ? -ESRCH : -EPERM; + } + try { + return Signal(handle, signal); + } finally { + CloseHandle(handle); + } + + default: + return -EINVAL; + } + } + + /// KILL or 0 to an open process handle: 0, or -errno. + private static int Signal(IntPtr handle, int signal) { + uint status; + if (!GetExitCodeProcess(handle, out status)) { + return -Errno(Marshal.GetLastWin32Error()); + } + if (status != STILL_ACTIVE) { + return -ESRCH; + } + if (signal == SIGKILL && !TerminateProcess(handle, 0)) { + return -EPERM; + } + return 0; + } + + #endregion + #region __waitpid__ private const int WNOHANG = 1; diff --git a/Src/StdLib/ironruby/ruby4.rb b/Src/StdLib/ironruby/ruby4.rb index ffb17f430..6b456eef6 100644 --- a/Src/StdLib/ironruby/ruby4.rb +++ b/Src/StdLib/ironruby/ruby4.rb @@ -5690,8 +5690,9 @@ class ClosedQueueError < StopIteration; end unless defined?(ClosedQueueError) class IO # Ruby defines NULL on IO; File inherits it. Defining it on File alone left - # IO::NULL undefined, which several specs and helpers reference. - NULL = "/dev/null" unless const_defined?(:NULL, false) + # IO::NULL undefined, which several specs and helpers reference. Windows has + # no /dev/null: there it is the NUL device, as in CRuby. + NULL = (File::ALT_SEPARATOR == "\\" ? "NUL" : "/dev/null") unless const_defined?(:NULL, false) end class << Dir From 29066bf609895dac49315a7213e84d88f8f1e0b3 Mon Sep 17 00:00:00 2001 From: Largo Date: Fri, 23 Oct 2026 16:29:57 +0200 Subject: [PATCH 5/8] JSON.parse: a raw control character in a string is a ParserError RFC 8259 has U+0000-U+001F escaped inside a string, and the json gem raises "invalid ASCII control character in string" for an unescaped one. IronRuby's parser took them as they were, so a document with a literal line break in a value parsed here and failed on CRuby - code that falls back on a ParserError for exactly that case never ran its fallback. JSON.parse(source, allow_control_characters: true), the gem's option since 2.18.0 (the API version this JSON reports), lets them through. DEL and escaped control characters stay fine. --- Src/IronRuby.Tests/RubyTests.cs | 1 + Src/IronRuby.Tests/Runtime/MiscTests.cs | 32 +++++++++++++++++++++++++ Src/Libraries/Json/Json.cs | 15 +++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Src/IronRuby.Tests/RubyTests.cs b/Src/IronRuby.Tests/RubyTests.cs index 66c2fbeda..bac7f7bc9 100644 --- a/Src/IronRuby.Tests/RubyTests.cs +++ b/Src/IronRuby.Tests/RubyTests.cs @@ -704,6 +704,7 @@ public Tests(Driver/*!*/ driver) { Time_OlsonZone1, Zlib_Streams1, Process_SpawnKill1, + Json_ControlCharacters1, Scenario_RubyThreads1, Scenario_YieldCodeGen, diff --git a/Src/IronRuby.Tests/Runtime/MiscTests.cs b/Src/IronRuby.Tests/Runtime/MiscTests.cs index 930f73648..4acded7f2 100644 --- a/Src/IronRuby.Tests/Runtime/MiscTests.cs +++ b/Src/IronRuby.Tests/Runtime/MiscTests.cs @@ -923,6 +923,38 @@ p got "); } + /// + /// A raw control character (U+0000-U+001F) inside a JSON string is a ParserError, as in the json gem; + /// JSON.parse took a document with a literal line break in a value as it was. The gem's + /// allow_control_characters option (2.18.0) lets them through. + /// + public void Json_ControlCharacters1() { + TestOutput(@" +load_assembly 'IronRuby.Libraries', 'IronRuby.StandardLibrary.Json' + +[10, 9, 1, 0, 31].each do |code| + src = '{""k"": ""a' + code.chr + 'b""}' + begin + JSON.parse(src) + p [code, :accepted] + rescue JSON::ParserError => e + p [code, e.message.start_with?('invalid ASCII control character in string')] + end +end +p JSON.parse('[""a' + 10.chr + 'b""]', :allow_control_characters => true) +# an escaped one, and DEL, are fine +p JSON.parse('[""a\nb"", ""' + 127.chr + '""]') == ['a' + 10.chr + 'b', 127.chr] +", @" +[10, true] +[9, true] +[1, true] +[0, true] +[31, true] +[""a\nb""] +true +"); + } + /// /// Zlib's streams and checksums. Windows has no libz; there they run on the zlib .NET ships /// (ClrZlib), so everything here holds on both: input CRuby compressed, round trips, and diff --git a/Src/Libraries/Json/Json.cs b/Src/Libraries/Json/Json.cs index f36d8ee36..0f62872af 100644 --- a/Src/Libraries/Json/Json.cs +++ b/Src/Libraries/Json/Json.cs @@ -291,6 +291,7 @@ public static object Parse(RubyContext/*!*/ context, object self, [DefaultProtoc var parser = new JsonParser(source.ConvertToString(), context); parser.SymbolizeNames = TruthyOption(context, options, "symbolize_names"); + parser.AllowControlCharacters = TruthyOption(context, options, "allow_control_characters"); object result = parser.ParseValue(0); parser.SkipWhitespace(); if (!parser.AtEnd) { @@ -301,7 +302,7 @@ public static object Parse(RubyContext/*!*/ context, object self, [DefaultProtoc /// /// One keyword of the options Hash JSON.parse takes. Only the options this parser - /// can honour are read: symbolize_names, and allow_comments, which needs no flag + /// can honour are read: symbolize_names, allow_control_characters, and allow_comments, which needs no flag /// because this parser has always skipped comments. An option it does not know is /// ignored rather than rejected, the way the json gem ignores unknown keys. /// @@ -352,6 +353,12 @@ internal sealed class JsonParser { /// internal bool SymbolizeNames { get; set; } + /// + /// JSON.parse(source, allow_control_characters: true) takes a raw newline, tab, ... inside + /// a string as it is. + /// + internal bool AllowControlCharacters { get; set; } + internal JsonParser(string/*!*/ source, RubyContext/*!*/ context) { _source = source; _context = context; @@ -494,6 +501,12 @@ private object ParseArray(int depth) { return builder.ToString(); } if (c != '\\') { + // RFC 8259: U+0000 through U+001F must be escaped inside a string. The json gem + // rejects them unless asked not to (2.18.0's allow_control_characters). + if (c < ' ' && !AllowControlCharacters) { + _position--; + throw new JsonParserError("invalid ASCII control character in string: '" + Rest + "'"); + } builder.Append(c); continue; } From 35922d628d1a8534a0392c8805d668a579f318eb Mon Sep 17 00:00:00 2001 From: Largo Date: Fri, 23 Oct 2026 16:37:10 +0200 Subject: [PATCH 6/8] net/http over TLS: diagnosis and proposed fix (not fixed) net/http is said to be undependable on Windows because IO.select on a socket is poll(2). Measured: plain sockets are fine (Socket.Select on Windows, and read_timeout fires); over TLS nothing times out - SSLSocket's read_nonblock, write_nonblock and connect_nonblock block in SslStream and never return :wait_readable, which is the only path net/protocol applies read_timeout on - and ca_file/cert_store/verify_callback are not consulted. Docs/net-http-over-tls.md has the diagnosis and a two-stage proposal: timeouts on the blocking SslStream calls plus the context's CAs first (small), real non-blocking reads through IoReadiness later. Util/net-http-tls-server.rb (CRuby) and Util/net-http-tls-check.rb reproduce both findings. --- Docs/net-http-over-tls.md | 132 ++++++++++++++++++++++++++++++++++++ Util/net-http-tls-check.rb | 55 +++++++++++++++ Util/net-http-tls-server.rb | 67 ++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 Docs/net-http-over-tls.md create mode 100644 Util/net-http-tls-check.rb create mode 100644 Util/net-http-tls-server.rb diff --git a/Docs/net-http-over-tls.md b/Docs/net-http-over-tls.md new file mode 100644 index 000000000..498b6ba45 --- /dev/null +++ b/Docs/net-http-over-tls.md @@ -0,0 +1,132 @@ +# net/http over TLS: diagnosis and proposed fix + +Status: diagnosed, not fixed. Measured on IronRuby 4.0.1, Windows 11, .NET 10.0.12, +2026-10-23. + +## The suspicion + +`net/http` has a reputation for being undependable on Windows, put down to `IO.select` +on a socket being `poll(2)` there. Applications have worked around it by calling .NET's +`HttpClient` instead. + +## What is actually the case + +`Util/net-http-tls-check.rb` against `Util/net-http-tls-server.rb` (CRuby) runs keep-alive +GETs and POSTs from 10 bytes to 300 KB, chunked bodies, a fresh connection per request, +and one request that has to end in `Net::ReadTimeout`: + +| | plain HTTP | HTTPS, `VERIFY_NONE` | HTTPS, `ca_file:` | +|---|---|---|---| +| CRuby 4.0 | no failures | no failures | no failures | +| IronRuby | no failures | **`read_timeout` never fires** | **every connection refused** | + +- **Plain sockets are fine.** `IO.select`, `IO#wait_readable` and + `read_nonblock` on a socket go through `Socket.Select` on Windows + (`Src/Libraries/Builtins/IoReadiness.cs`, `BlockWindows`), not `poll(2)`, and + `net/http`'s read timeout works. What *is* `poll(2)` on Windows is a pipe + (README, "What Windows does not have yet"); the suspicion mixes the two up. +- **Over TLS nothing times out.** A server that stops answering leaves the caller + blocked for ever: no `read_timeout`, no `write_timeout`, and no `open_timeout` + for the TLS handshake. For a service that calls an API in a loop, that is a hang. +- **`ca_file`, `ca_path`, `cert_store` and `verify_callback` are ignored.** Only the + system trust store counts, so a private CA or a self-signed test server cannot be + verified, only switched off with `VERIFY_NONE`. + +The data itself arrives intact over TLS: no corruption and no short reads were seen. + +## Why + +`OpenSSL::SSL::SSLSocket` (`Src/StdLib/ironruby/openssl/ssl.rb`) runs on +`OpenSSL::SSL::Transport__` (`Src/Libraries/OpenSSL/OpenSSL.cs`), an `SslStream` over +the TCP socket, which `Transport__.Create` switches to blocking mode. The header of +`ssl.rb` says what follows from that: `#read_nonblock`, `#write_nonblock` and +`#connect_nonblock` block, and never return `:wait_readable`/`:wait_writable`. + +`Net::BufferedIO#rbuf_fill` (`net/protocol.rb`) applies `read_timeout` only on the +`:wait_readable` path: + +```ruby +case rv = @io.read_nonblock(BUFSIZE, tmp, exception: false) +when String then ... +when :wait_readable + (io = @io.to_io).wait_readable(@read_timeout) or raise Net::ReadTimeout.new(io) +``` + +An `SSLSocket` never takes that path, so `read_nonblock` sits in `SslStream.Read` until +data comes. `ssl_socket_connect` (handshake under `open_timeout`) and `write_nonblock` +(under `write_timeout`) have the same shape. `Transport__.Validate` returns +`errors == SslPolicyErrors.None` and never looks at the context's CA settings. + +## Proposed fix + +There are two stages. The first is small and covers what a client needs; the second +gives real non-blocking I/O. + +### Stage 1: timeouts on the blocking calls, and the context's CAs (recommended next) + +1. **Read and write timeouts.** `Transport__` gets `read_timeout=` and `write_timeout=` + (milliseconds, -1 for none), which set `SslStream.ReadTimeout`/`WriteTimeout`. + They delegate to the inner `NetworkStream`, which means `Socket.ReceiveTimeout`/ + `SendTimeout`. A read that times out throws an `IOException` whose inner + `SocketException` is `TimedOut`. `Transport__.read`/`write` turn that into + `IO::TimeoutError`. An `SslStream` is unusable after a timed-out read, so the + transport marks itself broken and every later call raises. +2. **Where the timeout comes from.** IronRuby keeps a small override of + `Net::BufferedIO` in its own stdlib layer (`Src/StdLib/ironruby/`, loaded after + `net/protocol`). When `@io` is an IronRuby `SSLSocket`, `rbuf_fill` sets the + transport's read timeout from `@read_timeout` before the read and maps + `IO::TimeoutError` to `Net::ReadTimeout`. `write0` does the same with + `@write_timeout` and `Net::WriteTimeout`. The upstream file stays untouched, as + with the other stdlib shims. +3. **Handshake timeout.** `Transport__.connect` takes an optional timeout and calls + `AuthenticateAsClientAsync(options, cancellationToken)` with a + `CancellationTokenSource(timeout)`, waiting on the task. `ssl_socket_connect`'s + timeout reaches it through an override of `SSLSocket#connect_nonblock`, or through + `net/http` passing `open_timeout` (the override above can do that in `connect`). +4. **CA settings.** `Validate` builds the chain itself when the context has + `ca_file`, `ca_path` or `cert_store`: `X509ChainPolicy.TrustMode = + X509ChainTrustMode.CustomRootTrust` with those certificates in `CustomTrustStore`, + then `chain.Build`, plus the name check SslStream reported (`RemoteCertificateNameMismatch`). + `verify_callback`, when set, is called with the result and an `X509StoreContext` + holding the chain, and it has the last word, as in MRI. + +Risk: small. Blocking I/O stays as it is, only bounded. What it does not give is real +non-blocking `read_nonblock`, so `IO.select([ssl_socket])` still says "readable" only +when the TCP socket is. A reader that selects first and then reads can block in the +middle of a TLS record. `net/http` doesn't work that way. + +### Stage 2: non-blocking reads over SslStream + +- `Transport__.read_nonblock(length)` keeps at most one `ReadAsync` in flight. If none is + in flight, it starts one. If that completes at once (the plaintext was already + decrypted), it returns the data. Otherwise it returns `:wait_readable` and keeps the + task. The next call returns the task's result once it has completed. +- Readiness has to see that task: the in-flight read may already have drained the TCP + socket, so `Socket.Select` would wait although the data is there. `IoReadiness` + gets a per-`RubyIO` hook ("a pending read has completed") that `Block` checks + before it selects. It then waits on the task's handle together with the sockets, + the way `ReadinessWaiter` already stands in for pipes. `SSLSocket#to_io` keeps + returning the TCP socket, which carries the hook. +- `write_nonblock` and `connect_nonblock` follow the same pattern with `WriteAsync` and + `AuthenticateAsClientAsync`. + +Risk: medium. It touches `IO.select` for every caller, and the in-flight task has to be +cancelled cleanly on `close`. It is worth doing for servers and event loops (nio4r, +puma over TLS), not for a client. + +## How to test it + +- `Util/net-http-tls-check.rb` (needs CRuby for the server): after stage 1 the + `no_timeout` entry and, with `CA=1`, the refused connections are gone. +- For CI without CRuby: `IronRuby.Tests` can host the TLS server itself. Create a + certificate with `CertificateRequest` + `CreateSelfSigned`, accept on a loopback + `TcpListener`, `SslStream.AuthenticateAsServer`, and stall after the headers for the + timeout case. The Ruby side then runs `Net::HTTP` with `ca_file:` pointing at the + exported certificate and `read_timeout: 1` (the test needs `[Options(Prism = true)]` + to load `net/http`). + +## Until then + +An application that needs timeouts or its own CA over HTTPS is better served by .NET's +`HttpClient` until stage 1 lands. The reason is not `IO.select` being `poll(2)`; it is +that `net/http` over TLS has no timeouts and ignores `ca_file`. diff --git a/Util/net-http-tls-check.rb b/Util/net-http-tls-check.rb new file mode 100644 index 000000000..70e143b98 --- /dev/null +++ b/Util/net-http-tls-check.rb @@ -0,0 +1,55 @@ +# Net::HTTP over plain TCP and over TLS against Util/net-http-tls-server.rb: keep-alive +# GETs and POSTs of several sizes, chunked bodies, fresh connections, and a read_timeout +# that has to fire. See Docs/net-http-over-tls.md. +# +# ruby Util/net-http-tls-server.rb 18080 18443 # CRuby, in another terminal +# ir.cmd Util/net-http-tls-check.rb 18080 18443 40 # or ./ir.sh +# +# CA=1 verifies against the server's certificate (ca_file) instead of VERIFY_NONE. +# IronRuby 4.0.1: http has no failures; https reports no_timeout (the read_timeout never +# fires), and with CA=1 every connection is refused (ca_file is not consulted). +require "net/http" +require "openssl" +require "tmpdir" +http_port, https_port, rounds = ARGV.map(&:to_i) +rounds = 20 if rounds.zero? +cert = File.join(Dir.tmpdir, "net-http-tls-check-cert.pem") +[[:http, http_port, false], [:https, https_port, true]].each do |label, port, tls| + fails = Hash.new(0) + t0 = Time.now + begin + Net::HTTP.start("127.0.0.1", port, use_ssl: tls, **(tls ? (ENV["CA"] ? {ca_file: cert} : {verify_mode: OpenSSL::SSL::VERIFY_NONE}) : {}), open_timeout: 5, read_timeout: 10) do |h| + rounds.times do |i| + size = [10, 1000, 20_000, 300_000][i % 4] + path = (i % 3 == 0) ? "/chunked?size=#{size}" : "/data?size=#{size}" + begin + r = h.get(path) + fails[:size] += 1 unless r.body.bytesize == size + r = h.post("/echo", "y" * size) + fails[:echo] += 1 unless r.body.bytesize == size + rescue Exception => e + fails["#{e.class}: #{e.message[0, 60]}"] += 1 + end + end + end + rescue Exception => e + fails["start #{e.class}: #{e.message[0, 80]}"] += 1 + end + # one fresh connection per request too + rounds.times do + begin + Net::HTTP.start("127.0.0.1", port, use_ssl: tls, **(tls ? (ENV["CA"] ? {ca_file: cert} : {verify_mode: OpenSSL::SSL::VERIFY_NONE}) : {}), open_timeout: 5, read_timeout: 10) { |h| h.get("/data?size=5000") } + rescue Exception => e + fails["fresh #{e.class}: #{e.message[0, 60]}"] += 1 + end + end + # a read timeout must fire + begin + Net::HTTP.start("127.0.0.1", port, use_ssl: tls, **(tls ? (ENV["CA"] ? {ca_file: cert} : {verify_mode: OpenSSL::SSL::VERIFY_NONE}) : {}), read_timeout: 1) { |h| h.get("/data?delay=3") } + fails[:no_timeout] += 1 + rescue Net::ReadTimeout + rescue Exception => e + fails["timeout #{e.class}"] += 1 + end + puts "#{label}: #{rounds} rounds in #{(Time.now - t0).round(1)}s, failures #{fails.inspect}" +end diff --git a/Util/net-http-tls-server.rb b/Util/net-http-tls-server.rb new file mode 100644 index 000000000..44b2a8856 --- /dev/null +++ b/Util/net-http-tls-server.rb @@ -0,0 +1,67 @@ +# Test server for Util/net-http-tls-check.rb: HTTP/1.1 on one port, HTTPS on another, +# with a throwaway self-signed certificate for 127.0.0.1 written to Dir.tmpdir +# (net-http-tls-check-cert.pem). Run it with CRuby: IronRuby has no SSLServer. +# +# ruby Util/net-http-tls-server.rb 18080 18443 +# +# GET /data?size=N[&delay=S] answers N bytes after S seconds; /chunked?size=N the same, +# chunked; POST /echo sends the body back. +require "socket" +require "openssl" +require "tmpdir" + +http_port, https_port = ARGV.map(&:to_i) + +key = OpenSSL::PKey::RSA.new(2048) +cert = OpenSSL::X509::Certificate.new +cert.version = 2 +cert.serial = 1 +cert.subject = cert.issuer = OpenSSL::X509::Name.parse("/CN=127.0.0.1") +cert.public_key = key.public_key +cert.not_before = Time.now - 60 +cert.not_after = Time.now + 86400 +ef = OpenSSL::X509::ExtensionFactory.new(cert, cert) +cert.add_extension(ef.create_extension("subjectAltName", "IP:127.0.0.1")) +cert.sign(key, OpenSSL::Digest::SHA256.new) +File.write(File.join(Dir.tmpdir, "net-http-tls-check-cert.pem"), cert.to_pem) + +def serve(sock) + loop do + request = sock.gets("\r\n\r\n") or break + path = request[/\A\S+ (\S+)/, 1] + length = request[/^content-length: *(\d+)/i, 1].to_i + body_in = length > 0 ? sock.read(length) : "" + size = path[/size=(\d+)/, 1].to_i + sleep(path[/delay=([\d.]+)/, 1].to_f) + body = path.include?("echo") ? body_in : ("x" * (size > 0 ? size : 100)) + if path.include?("chunked") + sock.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n" + body.scan(/.{1,1000}/m) { |c| sock.write "#{c.bytesize.to_s(16)}\r\n#{c}\r\n"; sock.flush } + sock.write "0\r\n\r\n" + else + sock.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\n\r\n#{body}" + end + sock.flush + break if request =~ /^connection: *close/i + end +rescue StandardError +ensure + sock.close rescue nil +end + +ctx = OpenSSL::SSL::SSLContext.new +ctx.cert = cert +ctx.key = key +plain = TCPServer.new("127.0.0.1", http_port) +tls = TCPServer.new("127.0.0.1", https_port) +ssl = OpenSSL::SSL::SSLServer.new(tls, ctx) +ssl.start_immediately = true +$stdout.puts "listening #{http_port} #{https_port}"; $stdout.flush +Thread.new { loop { s = plain.accept; Thread.new(s) { |c| serve(c) } } } +loop do + begin + s = ssl.accept + Thread.new(s) { |c| serve(c) } + rescue StandardError + end +end From e880957856bc4e71568bcb46241cfb6772bd0f2c Mon Sep 17 00:00:00 2001 From: Largo Date: Mon, 26 Oct 2026 13:58:34 +0100 Subject: [PATCH 7/8] zlib on .NET 8: keep inflating while output is still coming Raw inflate on Windows / .NET 8 stopped after the first 16 KB: the run loop ends an inflate run once avail_in is 0, which with libz means it has nothing more to give. .NET 8's zlib is Intel's fork, which reads all of the input ahead into its bit buffer, so with the output full and no input left it can still have output to hand out. A zlib or gzip stream hid this, because its trailer was still unread. On .NET's zlib the loop asks again; when inflate really is done it leaves the fresh buffer untouched and the avail_out check ends the run. libz keeps the old condition. Zlib_Streams1 caught it on CI (windows-latest net8.0); with this its output matches CRuby's on .NET 8, and ruby/spec library/zlib there has 0 failures and the 3 NotImplementedErrors. --- Src/Libraries/Zlib/zlib.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Src/Libraries/Zlib/zlib.cs b/Src/Libraries/Zlib/zlib.cs index d7d59c4b2..631347c4f 100644 --- a/Src/Libraries/Zlib/zlib.cs +++ b/Src/Libraries/Zlib/zlib.cs @@ -740,10 +740,14 @@ private object RunNoLock(byte[]/*!*/ data, int flush, BlockParam block, out bool if (z.avail_out > 0) { break; } - if (z.avail_in == 0 && _isInflate) { + if (z.avail_in == 0 && _isInflate && !LibZ.UseClrZlib) { // inflate() answers Z_BUF_ERROR once it is out of input; deflate() // can still have output pending in its own state, so only inflate - // gets to stop here. + // gets to stop here. Not on .NET's zlib: Intel's fork in .NET 8 + // reads all the input ahead into its bit buffer, so with the output + // full and nothing left to read it can still have output to give. + // It is asked again, and when it really is done it leaves the fresh + // buffer untouched, which the avail_out check above ends on. break; } } From 567fa76b2781c0463c00e7072a8db89bce6abfbe Mon Sep 17 00:00:00 2001 From: Largo Date: Mon, 26 Oct 2026 13:59:03 +0100 Subject: [PATCH 8/8] README: library/zlib on Windows has the 2 byte-exact failures on .NET 10 only --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9076746e2..72c14aa25 100644 --- a/README.md +++ b/README.md @@ -544,13 +544,13 @@ Where the Windows errors are: `core/file` (113) and `core/filetest` (8) are syml `chmod`/`umask` and the other Unix file metadata; `core/process` (36 failures, 15 errors) is `fork`, process groups, uid/gid and signals; `library/win32ole` (114) and `library/readline` (25) are libraries IronRuby does not implement and whose specs only run on Windows; -`library/zlib` (2 failures, 3 errors) is the zlib below. `core/objectspace` needs +`library/zlib` (3 errors, and 2 failures on .NET 10) is the zlib below. `core/objectspace` needs `-X:ObjectSpace`, as on Linux. **`zlib`** on Windows runs on the zlib inside .NET itself (`System.IO.Compression.Native`: zlib-ng from .NET 9 on, Intel's zlib in .NET 8), since there is no libz to bind to. It -inflates and deflates, gzip included, but its compressed bytes are that compressor's, not -classic zlib's (the 2 `library/zlib` failures compare exact bytes), and what .NET does not +inflates and deflates, gzip included, but its compressed bytes are that compressor's: zlib-ng's +differ from classic zlib's (the 2 `library/zlib` failures on .NET 10 compare exact bytes), and what .NET does not export raises `NotImplementedError`: `set_dictionary`, `Deflate#params` once data has gone in, and `Inflate#sync_point?` (the 3 errors).