Broken String patterns, Thread.stop alone, and MRI's Windows behaviour for links and stubs

Src/IronRuby.Tests/RubyTests.cs

@@ -857,6 +857,12 @@ public Tests(Driver/*!*/ driver) {
                 File_StatPredicatesToIo2,
                 IO_SysOpenArguments1,
                 IO_SysOpenPermission1,
+                String_BrokenPattern1,
+                Thread_StopAlone1,
+                Thread_HostMainThread1,
+                Thread_HostMainThread2,
+                Windows_NotImplementedAndLinks1,
+                Windows_SymLinks1,
                 Range_IntegerFastPaths1,
                 Io_BuiltinReaders1,
                 Io_LineReaders2,

Src/Libraries/Builtins/Dir.cs

@@ -160,10 +160,17 @@ private static object ChangeDirectory(PlatformAdaptationLayer/*!*/ pal, string/*
             }
         }
 
+        // On Windows these four are MRI's rb_f_notimplement stubs (no chroot, fchdir, dirfd or
+        // fdopendir there): NotImplementedError, and #respond_to? denies them.
+        private static Exception/*!*/ Unimplemented(string/*!*/ function) {
+            return new NotImplementedError(function + "() function is unimplemented on this machine");
+        }
+
         [RubyMethod("chroot", RubyMethodAttributes.PublicSingleton)]
+        [RubyNotImplemented(OnWindowsOnly = true)]
         public static int ChangeRoot(ConversionStorage<MutableString>/*!*/ toPath, RubyClass/*!*/ self, object dir) {
             if (!Posix.IsAvailable) {
-                throw new InvalidOperationException();
+                throw Unimplemented("chroot");
             }
             var path = Protocols.CastToPath(toPath, dir);
             string strPath = self.Context.DecodePath(path);
@@ -180,9 +187,10 @@ public static int ChangeRoot(ConversionStorage<MutableString>/*!*/ toPath, RubyC
         /// lasts for the block only, and the block's value is the result.
         /// </summary>
         [RubyMethod("fchdir", RubyMethodAttributes.PublicSingleton)]
+        [RubyNotImplemented(OnWindowsOnly = true)]
         public static object ChangeDirectoryToDescriptor(BlockParam block, RubyClass/*!*/ self, [DefaultProtocol]int fd) {
             if (!Posix.IsAvailable) {
-                throw new NotImplementedException("fchdir");
+                throw Unimplemented("fchdir");
             }
 
             var pal = self.Context.Platform;
@@ -509,15 +517,17 @@ public static void Close(RubyDir/*!*/ self) {
         /// is opened the first time it is asked for and closed with the Dir.
         /// </summary>
         [RubyMethod("fileno")]
+        [RubyNotImplemented(OnWindowsOnly = true)]
         public static int FileNo(RubyDir/*!*/ self) {
             self.ThrowIfClosed();
             return self.GetFileDescriptor();
         }
 
         [RubyMethod("for_fd", RubyMethodAttributes.PublicSingleton)]
+        [RubyNotImplemented(OnWindowsOnly = true)]
         public static RubyDir/*!*/ ForFileDescriptor(RubyClass/*!*/ self, [DefaultProtocol]int fd) {
             if (!Posix.IsAvailable) {
-                throw new NotImplementedException("Dir.for_fd");
+                throw Unimplemented("for_fd");
             }
 
             // There is no fdopendir here, so the descriptor is turned back into the path it
@@ -661,7 +671,7 @@ private int GetFileDescriptor() {
                 return _fd;
             }
             if (!Posix.IsAvailable) {
-                throw new NotImplementedException("Dir#fileno");
+                throw Unimplemented("fileno");
             }
 
             string path = ImmediateClass.Context.DecodePath(_dirName);

Src/Libraries/Builtins/MutableStringOps.cs

@@ -932,6 +932,11 @@ public static MutableString RemoveSubstringInPlace(MutableString/*!*/ self, [Not
 
             // CLR indices of the character representations: a byte-level operation (#start_with?,
             // #getbyte) may have left either one as bytes.
+            // rb_str_index: incompatible encodings are an error before anything is searched
+            self.RequireCompatibleEncoding(searchStr);
+            if (IsBrokenNeedle(searchStr)) {
+                return null;
+            }
             self.PrepareForCharacterRead();
             searchStr.PrepareForCharacterRead();
             int index = self.IndexOf(searchStr);
@@ -1068,7 +1073,8 @@ public static MutableString GetSubstring(ConversionStorage<int>/*!*/ fixnumCast,
         [RubyMethod("[]")]
         [RubyMethod("slice")]
         public static MutableString GetSubstring(MutableString/*!*/ self, [NotNull]MutableString/*!*/ searchStr) {
-            return (self.IndexOf(searchStr) != -1) ? searchStr.CloneDerived() : null;
+            self.RequireCompatibleEncoding(searchStr);
+            return (!IsBrokenNeedle(searchStr) && self.IndexOf(searchStr) != -1) ? searchStr.CloneDerived() : null;
         }
 
         [RubyMethod("[]")]
@@ -1284,9 +1290,10 @@ public static MutableString ReplaceSubstring(MutableString/*!*/ self,
             [NotNull]MutableString/*!*/ substring, [DefaultProtocol, NotNull]MutableString/*!*/ value) {
 
             // see RemoveSubstringInPlace(self, searchStr)
+            self.RequireCompatibleEncoding(substring);
             self.PrepareForCharacterRead();
             substring.PrepareForCharacterRead();
-            int index = self.IndexOf(substring);
+            int index = IsBrokenNeedle(substring) ? -1 : self.IndexOf(substring);
             if (index == -1) {
                 throw RubyExceptions.CreateIndexError("string not matched");
             }
@@ -3124,8 +3131,8 @@ public static object BlockReplaceFirst(ConversionStorage<MutableString>/*!*/ tos
 
             object blockResult;
             MutableString result;
-            // TODO:
-            var regex = new RubyRegex(MutableString.CreateMutable(Regex.Escape(matchString.ToString()), matchString.Encoding), RubyRegexOptions.NONE);
+            // The same literal pattern the String forms without a block get from the Regexp conversion.
+            var regex = RubyRegex.FromLiteralPattern(matchString);
 
             return BlockReplaceFirst(tosConversion, scope, self, block, regex, out blockResult, out result) ? blockResult : (result ?? self.CloneDerived());
         }
@@ -3137,8 +3144,8 @@ public static object BlockReplaceAll(ConversionStorage<MutableString>/*!*/ tosCo
 
             object blockResult;
             MutableString result;
-            // TODO:
-            var regex = new RubyRegex(MutableString.CreateMutable(Regex.Escape(matchString.ToString()), matchString.Encoding), RubyRegexOptions.NONE);
+            // The same literal pattern the String forms without a block get from the Regexp conversion.
+            var regex = RubyRegex.FromLiteralPattern(matchString);
 
             self.TrackChanges();
             object r = BlockReplaceAll(tosConversion, scope, self, block, regex, out blockResult, out result) ? blockResult : (result ?? self.CloneDerived());
@@ -3276,25 +3283,62 @@ public static object ReplaceFirst(RubyScope/*!*/ scope, MutableString/*!*/ self,
             throw RubyExceptions.CreateArgumentError("wrong number of arguments (given 1, expected 2)");
         }
 
+        // The pattern is converted - and a String one checked (RubyRegex.FromLiteralPattern) - when
+        // the Enumerator runs, not when it is made: MRI's str_gsub returns the Enumerator before
+        // get_pat_quoted, so "abc".gsub("\xFF") is an Enumerator whose #to_a raises.
         [RubyMethod("gsub")]
-        public static object ReplaceAll(ConversionStorage<MutableString>/*!*/ tosConversion, RubyScope/*!*/ scope,
-            MutableString/*!*/ self, [DefaultProtocol, NotNull]RubyRegex/*!*/ pattern) {
+        //
+        // The binder may pick these for a call with a block too (an object parameter needs no
+        // conversion), so they do the work at once when there is one, as MRI does.
+        public static object ReplaceAll(ConversionStorage<MutableString>/*!*/ tosConversion, ConversionStorage<MutableString>/*!*/ stringTryCast,
+            RubyScope/*!*/ scope, BlockParam block, MutableString/*!*/ self, object pattern) {
+            if (block != null) {
+                return BlockReplaceAll(tosConversion, scope, block, self, ToPattern(stringTryCast, pattern));
+            }
             scope.MarkCaptured();
-            return new Enumerator((_, block) => BlockReplaceAll(tosConversion, scope, block, self, pattern));
+            return new Enumerator((_, b) => BlockReplaceAll(tosConversion, scope, b, self, ToPattern(stringTryCast, pattern)));
         }
 
         [RubyMethod("gsub!")]
-        public static object ReplaceAllInPlace(ConversionStorage<MutableString>/*!*/ tosConversion, RubyScope/*!*/ scope,
-            MutableString/*!*/ self, [DefaultProtocol, NotNull]RubyRegex/*!*/ pattern) {
+        public static object ReplaceAllInPlace(ConversionStorage<MutableString>/*!*/ tosConversion, ConversionStorage<MutableString>/*!*/ stringTryCast,
+            RubyScope/*!*/ scope, BlockParam block, MutableString/*!*/ self, object pattern) {
+            // rb_str_gsub_bang checks before anything else, the Enumerator included
+            self.RequireNotFrozen();
+            if (block != null) {
+                return BlockReplaceInPlace(tosConversion, scope, block, self, ToPattern(stringTryCast, pattern), true);
+            }
             scope.MarkCaptured();
-            return new Enumerator((_, block) => BlockReplaceInPlace(tosConversion, scope, block, self, pattern, true));
+            return new Enumerator((_, b) => BlockReplaceInPlace(tosConversion, scope, b, self, ToPattern(stringTryCast, pattern), true));
+        }
+
+        // MRI's get_pat_quoted: a Regexp as it is, a String (or what #to_str makes one) as a
+        // literal pattern, anything else Check_Type's TypeError.
+        private static RubyRegex/*!*/ ToPattern(ConversionStorage<MutableString>/*!*/ stringTryCast, object pattern) {
+            var regex = pattern as RubyRegex;
+            if (regex != null) {
+                return regex;
+            }
+            MutableString str = Protocols.TryCastToString(stringTryCast, pattern);
+            if (str == null) {
+                throw RubyExceptions.CreateUnexpectedTypeError(stringTryCast.Context, pattern, "Regexp");
+            }
+            return RubyRegex.FromLiteralPattern(str);
         }
 
         #endregion
 
 
         #region index, rindex
 
+        /// <summary>
+        /// MRI's rb_strseq_index and rb_str_rindex give up on a needle whose bytes are not valid in
+        /// its encoding: not found, rather than a search for the bytes. So do #index, #rindex,
+        /// #byteindex, #byterindex, #include?, #[], #slice, #slice!, #[]=, #partition and #rpartition.
+        /// </summary>
+        private static bool IsBrokenNeedle(MutableString/*!*/ needle) {
+            return needle.ContainsInvalidCharacters();
+        }
+
         // encoding aware
         [RubyMethod("index")]
         public static object Index(MutableString/*!*/ self, 
@@ -3305,6 +3349,9 @@ public static object Index(MutableString/*!*/ self,
             }
 
             self.RequireCompatibleEncoding(substring);
+            if (IsBrokenNeedle(substring)) {
+                return null;
+            }
             substring.PrepareForCharacterRead();
 
             int result = self.IndexOf(substring, self.ToClrIndex(start));
@@ -3351,6 +3398,9 @@ public static object LastIndexOf(MutableString/*!*/ self,
             }
 
             self.RequireCompatibleEncoding(substring);
+            if (IsBrokenNeedle(substring)) {
+                return null;
+            }
             // A substring with a pair cannot occur in a receiver without one, so its CLR length -
             // no scan of a fresh literal needed - is good enough for the checks below then.
             int subCharCount = self.KnowsCharIndexIsClrIndex

Src/Libraries/Builtins/RubyRegexOps.cs

@@ -96,27 +96,11 @@ internal static bool NormalizeGroupIndex(ref int index, int groupCount) {
         public static RubyRegex/*!*/ Create(RubyClass/*!*/ self,
             [DefaultProtocol, NotNull]MutableString/*!*/ pattern, [Optional]object options, [DefaultProtocol, Optional]MutableString encoding) {
 
-            // MRI rejects a pattern that is not valid in its own encoding before compiling it.
-            if (pattern.Encoding != RubyEncoding.Binary && pattern.ContainsInvalidCharacters()) {
-                throw new RegexpError("invalid multibyte character: /" + EscapeNonAsciiBytes(pattern) + "/");
-            }
+            RubyRegex.RequireValidPattern(pattern);
             double? timeout = TakeTimeout(self.Context, ref options);
             return new RubyRegex(pattern, MakeOptions(self.Context, options, encoding)) { Timeout = timeout };
         }
 
-        // For an error message about bytes that do not decode: ASCII as is, the rest as \xHH.
-        private static string/*!*/ EscapeNonAsciiBytes(MutableString/*!*/ str) {
-            var result = new System.Text.StringBuilder();
-            foreach (byte b in str.ToByteArray()) {
-                if (b < 0x80) {
-                    result.Append((char)b);
-                } else {
-                    result.Append("\\x").Append(b.ToString("X2", System.Globalization.CultureInfo.InvariantCulture));
-                }
-            }
-            return result.ToString();
-        }
-
         [RubyConstructor]
         public static RubyRegex/*!*/ Create(RubyClass/*!*/ self, DynamicNull pattern, [Optional]object options, [Optional]object encoding) {
             throw RubyExceptions.CreateImplicitConversionError("nil", "String");

Src/Libraries/Builtins/ThreadGroup.cs

@@ -62,8 +62,8 @@ public static bool IsEnclosed([NotNull]ThreadGroup/*!*/ self) {
         }
 
         [RubyMethod("list")]
-        public static RubyArray/*!*/ List([NotNull]ThreadGroup/*!*/ self) {
-            ThreadOps.RubyThreadInfo[] threads = ThreadOps.RubyThreadInfo.Threads;
+        public static RubyArray/*!*/ List(RubyContext/*!*/ context, [NotNull]ThreadGroup/*!*/ self) {
+            ThreadOps.RubyThreadInfo[] threads = ThreadOps.RubyThreadInfo.GetRubyThreads(context);
             RubyArray result = new RubyArray(threads.Length);
             foreach (ThreadOps.RubyThreadInfo threadInfo in threads) {
                 Thread thread = threadInfo.Thread;

Src/Libraries/Builtins/ThreadOps.Fiber.cs

@@ -229,6 +229,11 @@ public object Handle {
                 }
             }
 
+            /// <summary>The thread this fiber belongs to, which is Thread.current inside it.</summary>
+            internal Thread/*!*/ Owner {
+                get { return _owner; }
+            }
+
             internal FiberCore(RubyContext context, object handle, Thread owner, bool root) {
                 _mailbox = new FiberMailbox();
                 _owner = owner;
@@ -264,6 +269,11 @@ private void Activate() {
                 var info = _threadInfo ?? (_threadInfo = RubyThreadInfo.FromThread(Thread.CurrentThread));
                 info[_currentKey] = handle;
                 info.FiberExecution = this;
+                if (!_root) {
+                    // set before it is ever cleared again, so that no moment between two fibers -
+                    // FiberExecution gone, not yet parked - counts this thread as a Ruby thread
+                    info.IsFiberWorker = true;
+                }
                 RubyUtils.SetFiberOwnerThread(_owner);
                 _mailbox.GetOwnerInfo(_owner).ActiveFiberThread = Thread.CurrentThread;
             }