Roundtrip fixes

Datamodel.NET/Arrays.cs

@@ -82,15 +82,19 @@ public void CopyTo(T[] array, int offset)
 
         bool ICollection<T>.IsReadOnly { get { return false; } }
 
-        public bool IsFixedSize => throw new NotImplementedException();
+        public bool IsFixedSize => false;
 
-        public bool IsReadOnly => throw new NotImplementedException();
+        public bool IsReadOnly => false;
 
-        public bool IsSynchronized => throw new NotImplementedException();
+        public bool IsSynchronized => false;
 
-        public object SyncRoot => throw new NotImplementedException();
+        public object SyncRoot => Inner;
 
-        object? IList.this[int index] { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
+        object? IList.this[int index]
+        {
+            get => this[index];
+            set => this[index] = value is null ? throw new InvalidOperationException("Trying to set a null object") : (T)value;
+        }
 
         public bool Remove(T item) => Inner.Remove(item);
 

Datamodel.NET/AttributeList.cs

@@ -282,17 +282,10 @@ public virtual object? this[string name]
                 {
                     if (prop.CanWrite)
                     {
-                        // were actually fine with this being null, it will just set the value to null
-                        // but need to check so the type check doesn't fail if it is null
-                        if (value != null)
+                        // null is fine, it will just set the value to null
+                        if (value != null && !prop.PropertyType.IsInstanceOfType(value))
                         {
-                            var valueType = value.GetType();
-
-                            // types must be equal, or a superclass
-                            if (prop.PropertyType != typeof(Element) && valueType.IsSubclassOf(prop.PropertyType))
-                            {
-                                throw new InvalidDataException($"class property '{prop.Name}' with type '{prop.PropertyType}' does not match the type '{valueType}' of the value being set, this is likely a mismatch between the real class and the class from the datamodel");
-                            }
+                            throw new InvalidDataException($"class property '{prop.DeclaringType!.Name}.{prop.Name}' with type '{prop.PropertyType}' can not hold a value of type '{value.GetType()}' (attribute '{name}'), this is likely a mismatch between the real class and the class from the datamodel");
                         }
 
                         prop.SetValue(this, value);
@@ -308,16 +301,16 @@ public virtual object? this[string name]
                             if (existingArray.Count == 0)
                             {
                                 existingArray.AddRange(incomingArray);
-                                return;
                             }
                             else
                             {
                                 throw new InvalidOperationException($"Attribute '{name}' modifies property {prop.DeclaringType!.Name}.{prop.Name}, which is write only and can't be replaced.");
                             }
                         }
-
-
-                        throw new InvalidDataException("Property of deserialisation class must be writeable, make sure it's public and has a public setter");
+                        else
+                        {
+                            throw new InvalidDataException($"Property '{prop.DeclaringType!.Name}.{prop.Name}' of deserialisation class must be writeable, make sure it's public and has a public setter");
+                        }
                     }
 
                     return;
@@ -432,6 +425,9 @@ public int Count
         /// </summary>
         public object SyncRoot { get { return Attribute_ChangeLock; } }
 
+        /// <summary>
+        /// Enumerates every attribute to be written by a codec: class properties first, in declaration order, followed by the plain attributes in the order they were added.
+        /// </summary>
         public IEnumerable<AttrKVP> GetAllAttributesForSerialization()
         {
             foreach (var attr in GetPropertyBasedAttributes(useSerializationName: true))

Datamodel.NET/Codecs/Binary.cs

@@ -21,6 +21,11 @@
         static readonly Dictionary<int, Type?[]> SupportedAttributes = [];
         BinaryReader? Reader;
 
+        /// <summary>
+        /// Elements in the order the stream declares them. Element references are indices into this list, which must not change for deferred loading.
+        /// </summary>
+        readonly List<Element> ElementIndex = [];
+
         /// <summary>
         /// The number of Datamodel binary ticks in one second. Used to store TimeSpan values.
         /// </summary>
@@ -48,7 +53,8 @@
 
         static byte TypeToId(Type type, int version)
         {
-            bool array = Datamodel.IsDatamodelArrayType(type);
+            // a byte[] is a "binary" blob, distinct from a "uint8_array" (Array<byte>) in encoding version 9
+            bool array = type != typeof(byte[]) && Datamodel.IsDatamodelArrayType(type);
             var search_type = array ? Datamodel.GetArrayInnerType(type) : type;
 
             if (array && search_type == typeof(byte) && !SupportedAttributes[version].Contains(typeof(byte)))
@@ -166,6 +172,19 @@
                     Scraped = [];
 
                     ScrapeElement(dm.Root);
+
+                    // the prefix attributes are also written as a regular element in version 9
+                    if (EncodingVersion >= 9 && dm.PrefixAttributes.Count > 0)
+                    {
+                        AddString(string.Empty);
+                        AddString(PrefixElementClass);
+                        foreach (var attr in dm.PrefixAttributes)
+                        {
+                            AddString(attr.Key);
+                            if (attr.Value is string stringValue)
+                                AddString(stringValue);
+                        }
+                    }
                 }
             }
 
@@ -322,7 +341,10 @@
                 return dm.AllElements[id] ?? new Element(dm, id);
             }
 
-            return dm.AllElements[index];
+            if (index < 0 || index >= ElementIndex.Count)
+                throw new CodecException($"Element index {index} is out of range.");
+
+            return ElementIndex[index];
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -375,8 +397,7 @@
 
         public Datamodel Decode(string encoding, int encoding_version, string format, int format_version, Stream stream, DeferredMode defer_mode, ReflectionParams reflectionParams)
         {
-            var elementFactoryTypes = CodecUtilities.GetIElementFactoryClasses();
-            var elementFactory = (IElementFactory)Activator.CreateInstance(elementFactoryTypes.First());
+            var resolver = new ElementTypeResolver(reflectionParams);
 
             stream.Seek(0, SeekOrigin.Begin);
             while (true)
@@ -416,16 +437,18 @@
                 var id_bits = Reader.ReadBytes(16);
                 var id = new Guid(BitConverter.IsLittleEndian ? id_bits : id_bits.Reverse().ToArray());
 
-                if (!CodecUtilities.TryConstructCustomElement(elementFactory, reflectionParams, dm, type, name, id, out _))
+                if (!CodecUtilities.TryConstructCustomElement(resolver, dm, type, name, id, out var elem))
                 {
                     // note: constructing an element, adds it to the datamodel.AllElements
-                    _ = new Element(dm, name, id, type);
+                    elem = new Element(dm, name, id, type);
                 }
+
+                ElementIndex.Add(elem!);
             }
 
 
             // read attributes (or not, if we're deferred)
-            foreach (var elem in dm.AllElements.ToArray())
+            foreach (var elem in ElementIndex)
             {
                 // assert if stub
                 Debug.Assert(!elem.Stub);
@@ -447,9 +470,24 @@
                 }
             }
 
+            // version 9 also stores the prefix attributes as an unreferenced element right after the root, fold it back in
+            if (EncodingVersion >= 9 && dm.PrefixAttributes.Count > 0 && dm.AllElements.Count > 1)
+            {
+                var duplicate = dm.AllElements[1];
+
+                if (duplicate != null && !duplicate.Stub && duplicate.ClassName == PrefixElementClass && duplicate.Name.Length == 0
+                    && duplicate.Keys.SequenceEqual(dm.PrefixAttributes.Keys))
+                {
+                    dm.PrefixElementId = duplicate.ID;
+                    dm.AllElements.RemoveUnreferenced(duplicate);
+                }
+            }
+
             return dm;
         }
 
+        const string PrefixElementClass = "DmElement";
+
         int EncodingVersion;
 
         public object? DeferredDecodeAttribute(Datamodel dm, long offset)
@@ -468,7 +506,7 @@
             var types = IdToType(reader.ReadByte());
 
             if (types.Item2 == null)
                 return ReadValue(dm, TypeMap[types.Item1.TypeHandle], EncodingVersion < 4 || prefix, reader);
             else
             {
                 var count = reader.ReadInt32();
@@ -553,7 +591,7 @@
         readonly struct Encoder
         {
             readonly Dictionary<Element, int> ElementIndices;
-            readonly List<Element> ElementOrder;
+            readonly List<AttributeList> ElementOrder;
             readonly BinaryWriter Writer;
             readonly StringDictionary StringDict;
             readonly Datamodel Datamodel;
@@ -579,20 +617,30 @@
                 Writer.Write(string.Format(CodecUtilities.HeaderPattern, "binary", EncodingVersion, Datamodel.Format, Datamodel.FormatVersion) + "\n");
 
                 if (EncodingVersion >= 9)
-                    Writer.Write(0); // Prefix elements
+                {
+                    WritePrefixAttributes();
+                }
 
                 StringDict.WriteSelf(Writer);
 
+                var hasPrefixElement = EncodingVersion >= 9 && Datamodel.PrefixAttributes.Count > 0;
+                var elementCount = CountChildren(Datamodel.Root, []) + (hasPrefixElement ? 1 : 0);
+                Writer.Write(elementCount);
+
+                var root = Datamodel.Root;
+                if (root != null && !root.Stub)
                 {
-                    var counter = new HashSet<Element>();
-                    var elementCount = CountChildren(Datamodel.Root, counter);
+                    WriteIndexEntry(root, root.ClassName, root.Name, root.ID);
 
-                    Writer.Write(elementCount);
+                    // the prefix attributes are also stored as an unreferenced element right after the root
+                    if (hasPrefixElement)
+                        WriteIndexEntry(Datamodel.PrefixAttributes, PrefixElementClass, string.Empty, Datamodel.PrefixElementId);
+
+                    WriteIndexChildren(root);
                 }
 
-                WriteIndex(Datamodel.Root);
-                foreach (var e in ElementOrder)
-                    WriteBody(e);
+                foreach (var body in ElementOrder)
+                    WriteBody(body);
             }
 
             int CountChildren(Element? elem, HashSet<Element> counter)
@@ -625,16 +673,26 @@
 
             void WriteIndex(Element? elem)
             {
-                if (elem is null || elem.Stub) return;
+                if (elem is null || elem.Stub || ElementIndices.ContainsKey(elem)) return;
+
+                WriteIndexEntry(elem, elem.ClassName, elem.Name, elem.ID);
+                WriteIndexChildren(elem);
+            }
 
-                ElementIndices[elem] = ElementIndices.Count;
-                ElementOrder.Add(elem);
+            void WriteIndexEntry(AttributeList body, string className, string name, Guid id)
+            {
+                if (body is Element elem)
+                    ElementIndices[elem] = ElementOrder.Count;
+                ElementOrder.Add(body);
 
-                StringDict.WriteString(elem.ClassName, Writer);
-                if (EncodingVersion >= 4) StringDict.WriteString(elem.Name, Writer);
-                else Writer.Write(elem.Name);
-                Writer.Write(elem.ID.ToByteArray());
+                StringDict.WriteString(className, Writer);
+                if (EncodingVersion >= 4) StringDict.WriteString(name, Writer);
+                else Writer.Write(name);
+                Writer.Write(id.ToByteArray());
+            }
 
+            void WriteIndexChildren(Element elem)
+            {
                 foreach (var attr in Context.Attributes[elem])
                 {
                     var child_elem = attr.Value as Element;
@@ -656,31 +714,63 @@
                 }
             }
 
-            void WriteBody(Element elem)
+            /// <summary>
+            /// Prefix attributes are stored as a list of prefix elements, each a list of name/typed value pairs.
+            /// Only the first prefix element is read back, so everything is written into a single one.
+            /// </summary>
+            void WritePrefixAttributes()
+            {
+                var prefixAttributes = Datamodel.PrefixAttributes.Where(attr => attr.Value != null).ToArray();
+
+                if (prefixAttributes.Length == 0)
+                {
+                    Writer.Write(0);
+                    return;
+                }
+
+                Writer.Write(1);
+                Writer.Write(prefixAttributes.Length);
+
+                foreach (var attr in prefixAttributes)
+                {
+                    Writer.Write(attr.Key);
+                    WriteTypedValue(attr.Value, raw_string: true);
+                }
+            }
+
+            void WriteBody(AttributeList elem)
             {
-                var attributesIterated = Context.Attributes[elem];
-                //Writer.Write(elem.Count);
+                var attributesIterated = elem is Element element ? Context.Attributes[element] : elem.GetAllAttributesForSerialization().ToArray();
                 Writer.Write(attributesIterated.Length);
                 foreach (var attr in attributesIterated)
                 {
                     StringDict.WriteString(attr.Key, Writer);
-                    var attr_type = attr.Value == null ? typeof(Element) : attr.Value.GetType();
-                    var attr_type_id = TypeToId(attr_type, EncodingVersion);
-                    Writer.Write(attr_type_id);
+                    WriteTypedValue(attr.Value, raw_string: false);
+                }
+            }
 
-                    if (attr.Value == null || !Datamodel.IsDatamodelArrayType(attr.Value.GetType()))
-                        WriteAttribute(attr.Value, false);
-                    else
-                    {
-                        var array = (System.Collections.IList)attr.Value;
-                        Writer.Write(array.Count);
-                        attr_type = Datamodel.GetArrayInnerType(array.GetType());
-                        foreach (var item in array)
-                            WriteAttribute(item, true);
-                    }
+            /// <summary>
+            /// Writes the type id of a value followed by the value itself, or by the item count and items for arrays.
+            /// </summary>
+            void WriteTypedValue(object? value, bool raw_string)
+            {
+                var attr_type = value == null ? typeof(Element) : value.GetType();
+                var attr_type_id = TypeToId(attr_type, EncodingVersion);
+                Writer.Write(attr_type_id);
+
+                if (value == null || value is byte[] || !Datamodel.IsDatamodelArrayType(attr_type))
+                {
+                    WriteAttribute(value, raw_string);
+                    return;
                 }
+
+                var array = (System.Collections.IList)value;
+                Writer.Write(array.Count);
+                foreach (var item in array)
+                    WriteAttribute(item, true);
             }
 
+            /// <param name="in_array">Whether the value is an array item or a prefix attribute, in which case strings are written inline rather than through the dictionary.</param>
             void WriteAttribute(object? value, bool in_array)
             {
                 if (value == null)