inabox/InABox.Core/Serialization.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using InABox.Clients;

namespace InABox.Core
{
    public enum SerializationFormat
    {
        Json,
        Binary
    }

    public class SerialisationException : Exception
    {
        public SerialisationException(string message): base(message) { }
    }

    public interface ISerializeBinary
    {
        public void SerializeBinary(CoreBinaryWriter writer);

        public void DeserializeBinary(CoreBinaryReader reader);
    }

    public static class Serialization
    {
        //private static JsonSerializerOptions? _serializerSettings;

        private static JsonSerializerOptions SerializerSettings(bool indented = true)
        {
            var serializerSettings = CreateSerializerSettings();
            serializerSettings.WriteIndented = indented;

            return serializerSettings;
        }

        public static JsonSerializerOptions CreateSerializerSettings(bool indented = true)
        {
            var settings = new JsonSerializerOptions
            {
                // DateParseHandling = DateParseHandling.DateTime,
                // DateFormatHandling = DateFormatHandling.IsoDateFormat,
                // DateTimeZoneHandling = DateTimeZoneHandling.RoundtripKind
            };

            settings.Converters.Add(new CoreTableJsonConverter());

            settings.Converters.Add(new FilterJsonConverter());
            settings.Converters.Add(new ColumnJsonConverter());
            settings.Converters.Add(new SortOrderJsonConverter());
            settings.Converters.Add(new UserPropertiesJsonConverter());
            settings.Converters.Add(new BaseObjectJSONConverter());

            settings.WriteIndented = indented; // ? Formatting.Indented : Formatting.None;
            return settings;
        }

        public static string Serialize(object? o, bool indented = false)
        {
            var json = JsonSerializer.Serialize(o, SerializerSettings(indented));
            return json;
        }

        public static void Serialize(object o, Stream stream, bool indented = false)
        {
            var settings = SerializerSettings(indented);
            JsonSerializer.Serialize(stream, o, settings);
            // using (var sw = new StreamWriter(stream))
            // {
            //     using (JsonWriter writer = new JsonTextWriter(sw))
            //     {
            //         var serializer = JsonSerializer.Create(settings);
            //         serializer.Serialize(writer, o);
            //     }
            // }
        }

        // public static void DeserializeInto(string json, object target)
        // {
        //     JsonConvert.PopulateObject(json, target, SerializerSettings());
        // }

        [return: MaybeNull]
        public static T Deserialize<T>(Stream? stream, bool strict = false)
        {
            if (stream == null)
                return default;

            try
            {
                var settings = SerializerSettings();
                return JsonSerializer.Deserialize<T>(stream, settings);
                // using var sr = new StreamReader(stream);
                // using JsonReader reader = new JsonTextReader(sr);
                // var serializer = JsonSerializer.Create(settings);
                // return serializer.Deserialize<T>(reader);
            }
            catch (Exception e)
            {
                if (strict)
                    throw;

                Logger.Send(LogType.Error, ClientFactory.UserID, $"Error in Deserialize<{typeof(T)}>(): {e.Message}");
                return default;
            }
        }

        public static object? Deserialize(Type type, Stream? stream)
        {
            if (stream == null)
                return null;

            object? result = null;
            var settings = SerializerSettings();
            result = JsonSerializer.Deserialize(stream, type, settings);
            // using (var sr = new StreamReader(stream))
            // {
            //     using (JsonReader reader = new JsonTextReader(sr))
            //     {
            //         var serializer = JsonSerializer.Create(settings);
            //         result = serializer.Deserialize(reader, type);
            //     }
            // }

            return result;
        }

        // [return: MaybeNull]
        // public static T Deserialize<T>(JToken obj, bool strict = false)
        // {
        //     var ret = default(T);
        //     try
        //     {
        //         var settings = SerializerSettings();
        //         var serializer = JsonSerializer.Create(settings);
        //         return obj.ToObject<T>();
        //     }
        //     catch (Exception)
        //     {
        //         if (strict)
        //         {
        //             throw;
        //         }
        //         if (typeof(T).IsArray)
        //         {
        //             ret = (T)(object)Array.CreateInstance(typeof(T).GetElementType(), 0);
        //         }
        //         else
        //         {
        //             ret = Activator.CreateInstance<T>();
        //         }
        //     }
        //
        //     return ret;
        // }

        [return: MaybeNull]
        public static T Deserialize<T>(string? json, bool strict = false) // where T : new()
        {
            var ret = default(T);
            if (string.IsNullOrWhiteSpace(json))
                return ret;

            try
            {
                var settings = SerializerSettings();
                //if (typeof(T).IsSubclassOf(typeof(BaseObject)))
                //{
                //    ret = Activator.CreateInstance<T>();
                //    (ret as BaseObject).SetObserving(false);
                //    JsonConvert.PopulateObject(json, ret, settings);
                //    (ret as BaseObject).SetObserving(true);
                //}
                //else 
                if (typeof(T).IsArray)
                {
                    ret = JsonSerializer.Deserialize<T>(json, settings);
                    //object o = Array.CreateInstance(typeof(T).GetElementType(), 0);
                    //ret = (T)o;
                }
                else
                {
                    ret = JsonSerializer.Deserialize<T>(json, settings);
                }
            }
            catch (Exception e)
            {
                if (strict)
                {
                    throw;
                }

                CoreUtils.LogException("", e);
                if (typeof(T).IsArray)
                {
                    ret = (T)(object)Array.CreateInstance(typeof(T).GetElementType(), 0);
                }
                else
                {
                    ret = (T)Activator.CreateInstance(typeof(T), true);
                }
            }

            return ret;
        }

        public static object? Deserialize(Type T, string json) // where T : new()
        {
            var ret = T.GetDefault();
            if (string.IsNullOrWhiteSpace(json))
                return ret;

            try
            {
                var settings = SerializerSettings();
                //if (typeof(T).IsSubclassOf(typeof(BaseObject)))
                //{
                //    ret = Activator.CreateInstance<T>();
                //    (ret as BaseObject).SetObserving(false);
                //    JsonConvert.PopulateObject(json, ret, settings);
                //    (ret as BaseObject).SetObserving(true);
                //}
                //else 
                if (T.IsArray)
                {
                    object o = Array.CreateInstance(T.GetElementType(), 0);
                    ret = o;
                }
                else
                {
                    ret = JsonSerializer.Deserialize(json, T, settings);
                }
            }
            catch (Exception)
            {
                ret = Activator.CreateInstance(T, true);
            }

            return ret;
        }

        #region Binary Serialization

        public static byte[] WriteBinary(this ISerializeBinary obj, BinarySerializationSettings settings)
        {
            using var stream = new MemoryStream();
            obj.SerializeBinary(new CoreBinaryWriter(stream, settings));

            return stream.ToArray();
        }
        public static void WriteBinary(this ISerializeBinary obj, Stream stream, BinarySerializationSettings settings)
        {
            obj.SerializeBinary(new CoreBinaryWriter(stream, settings));
        }

        public static T ReadBinary<T>(byte[] data, BinarySerializationSettings settings)
            where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), data, settings);
        public static T ReadBinary<T>(Stream stream, BinarySerializationSettings settings)
            where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), stream, settings);

        public static object ReadBinary(Type T, byte[] data, BinarySerializationSettings settings)
        {
            using var stream = new MemoryStream(data);
            return ReadBinary(T, stream, settings);
        }
        public static object ReadBinary(Type T, Stream stream, BinarySerializationSettings settings)
        {
            var obj = (Activator.CreateInstance(T) as ISerializeBinary)!;
            obj.DeserializeBinary(new CoreBinaryReader(stream, settings));
            return obj;
        }

        #endregion
    }

    public class CoreBinaryReader : BinaryReader
    {
        public BinarySerializationSettings Settings { get; set; }

        public CoreBinaryReader(Stream stream, BinarySerializationSettings settings) : base(stream)
        {
            Settings = settings;
        }
    }
    public class CoreBinaryWriter : BinaryWriter
    {
        public BinarySerializationSettings Settings { get; set; }

        public CoreBinaryWriter(Stream stream, BinarySerializationSettings settings) : base(stream)
        {
            Settings = settings;
        }
    }

    /// <summary>
    /// A class to maintain the consistency of serialisation formats across versions.
    /// The design of this is such that specific versions of serialisation have different parameters set,
    /// and the versions are maintained as static properties. Please keep the constructor private.
    /// </summary>
    /// <remarks>
    /// Note that <see cref="Latest"/> should always be updated to point to the latest version.
    /// <br/>
    /// Note also that all versions should have an entry in the <see cref="ConvertVersionString(string)"/> function.
    /// <br/>
    /// Also, if you create a new format, it would probably be a good idea to add a database update script to get all
    /// <see cref="IPackable"/> and <see cref="ISerializeBinary"/> properties and update the version of the format.
    /// (Otherwise, we'd basically be nullifying all data that is currently binary serialised.)
    /// </remarks>
    public class BinarySerializationSettings
    {
        
        /// <summary>
        /// Should the Info() call return RPC and Rest Ports?  This is
        /// To workaround a bug in RPCsockets that crash on large uploads
        /// </summary>
        /// <remarks>
        /// True in all serialization versions >= 1.2
        /// </remarks>
        public bool RPCClientWorkaround { get; set; }
        
        /// <summary>
        /// Should reference types include a flag for nullability? (Adds an extra boolean field for whether the value is null or not).
        /// </summary>
        /// <remarks>
        /// True in all serialisation versions >= 1.1.
        /// </remarks>
        public bool IncludeNullables { get; set; }
        
        public string Version { get; set; }

        public static BinarySerializationSettings Latest => V1_2;
        


        public static BinarySerializationSettings V1_0 = new BinarySerializationSettings("1.0")
        {
            IncludeNullables = false,
            RPCClientWorkaround = false
        };
        
        public static BinarySerializationSettings V1_1 = new BinarySerializationSettings("1.1")
        {
            IncludeNullables = true,
            RPCClientWorkaround = false
        };
        
        public static BinarySerializationSettings V1_2 = new BinarySerializationSettings("1.2")
        {
            IncludeNullables = true,
            RPCClientWorkaround = true
        };

        public static BinarySerializationSettings ConvertVersionString(string version) => version switch
        {
            "1.0" => V1_0,
            "1.1" => V1_1,
            "1.2" => V1_2,
            _ => V1_0
        };

        private BinarySerializationSettings(string version)
        {
            Version = version;
        }
    }

    public static class SerializationUtils
    {
        public static void Write(this BinaryWriter writer, Guid guid)
        {
            writer.Write(guid.ToByteArray());
        }
        public static Guid ReadGuid(this BinaryReader reader)
        {
            return new Guid(reader.ReadBytes(16));
        }

        public static void Write(this BinaryWriter writer, DateTime dateTime)
        {
            writer.Write(dateTime.Ticks);
        }
        public static DateTime ReadDateTime(this BinaryReader reader)
        {
            return new DateTime(reader.ReadInt64());
        }

        private static bool MatchType<T1>(Type t) => typeof(T1) == t;
        private static bool MatchType<T1,T2>(Type t) => (typeof(T1) == t) || (typeof(T2) == t);
        
        /// <summary>
        /// Binary serialize a bunch of different types of values. <see cref="WriteBinaryValue(CoreBinaryWriter, Type, object?)"/> and
        /// <see cref="ReadBinaryValue(CoreBinaryReader, Type)"/> are inverses of each other.
        /// </summary>
        /// <remarks>
        /// Handles <see cref="byte"/>[], <see cref="Array"/>s of serialisable values, <see cref="Enum"/>, <see cref="bool"/>, <see cref="string"/>,
        /// <see cref="Guid"/>, <see cref="byte"/>, <see cref="Int16"/>, <see cref="Int32"/>, <see cref="Int64"/>, <see cref="float"/>, <see cref="double"/>,
        /// <see cref="DateTime"/>, <see cref="TimeSpan"/>, <see cref="LoggablePropertyAttribute"/>, <see cref="IPackable"/>, <see cref="Nullable{T}"/>
        /// and <see cref="ISerializeBinary"/>.
        /// </remarks>
        /// <param name="writer"></param>
        /// <param name="type"></param>
        /// <param name="value"></param>
        /// <exception cref="Exception">If an object of <paramref name="type"/> is unable to be serialized.</exception>
        public static void WriteBinaryValue(this CoreBinaryWriter writer, Type type, object? value)
        {
            value ??= CoreUtils.GetDefault(type);
            
            if (value == null)
            {
                if (MatchType<string>(type))
                    writer.Write("");
                else if (writer.Settings.IncludeNullables && typeof(IPackable).IsAssignableFrom(type))
                    writer.Write(false);
                else if (writer.Settings.IncludeNullables && typeof(ISerializeBinary).IsAssignableFrom(type))
                    writer.Write(false);
                else if (Nullable.GetUnderlyingType(type) is Type t)
                    writer.Write(false);
                else if (MatchType<LoggablePropertyAttribute, object>(type))
                    writer.Write("");
                else
                    writer.Write(0);
            }
            
            else if (MatchType<byte[], object>(type) && value is byte[] bArray)
            {
                writer.Write(bArray.Length);
                writer.Write(bArray);
            }
            
            else if (type.IsArray && value is Array array)
            {
                var elementType = type.GetElementType();
                writer.Write(array.Length);
                foreach (var val1 in array)
                {
                    WriteBinaryValue(writer, elementType, val1);
                }
            }
            
            else if (type.IsEnum && value is Enum e)
            {
                var underlyingType = type.GetEnumUnderlyingType();
                WriteBinaryValue(writer, underlyingType, Convert.ChangeType(e, underlyingType));
            }
            
            else if (MatchType<bool, object>(type) && value is bool b)
            {
                writer.Write(b);
            }
            
            else if (MatchType<string, object>(type) && value is string str)
                writer.Write(str);

            else if (MatchType<Guid, object>(type) && value is Guid guid)
                writer.Write(guid);
            
            else if (MatchType<byte, object>(type) && value is byte i8)
                writer.Write(i8);
            
            else if (MatchType<Int16, object>(type) && value is Int16 i16)
                writer.Write(i16);
            
            else if (MatchType<Int32, object>(type) && value is Int32 i32)
                writer.Write(i32);
            
            else if (MatchType<Int64, object>(type) && value is Int64 i64)
                writer.Write(i64);
            
            else if (MatchType<float, object>(type) && value is float f32)
                writer.Write(f32);
            
            else if (MatchType<double, object>(type) && value is double f64)
                writer.Write(f64);
            
            else if (MatchType<DateTime, object>(type) && value is DateTime date)
                writer.Write(date.Ticks);
            
            else if (MatchType<TimeSpan, object>(type) && value is TimeSpan time)
                writer.Write(time.Ticks);
            
            else if (MatchType<LoggablePropertyAttribute, object>(type) && value is LoggablePropertyAttribute lpa)
                writer.Write(lpa.Format ?? string.Empty);
            
            else if (typeof(IPackable).IsAssignableFrom(type) && value is IPackable pack)
            {
                if (writer.Settings.IncludeNullables)
                    writer.Write(true);
                pack.Pack(writer);
            }
            
            else if (typeof(ISerializeBinary).IsAssignableFrom(type) && value is ISerializeBinary binary)
            {
                if (writer.Settings.IncludeNullables)
                    writer.Write(true);
                binary.SerializeBinary(writer);
            }
            
            else if (Nullable.GetUnderlyingType(type) is Type t)
            {
                writer.Write(true);
                writer.WriteBinaryValue(t, value);
            }
            
            else if (value is UserProperty userprop)
                WriteBinaryValue(writer, userprop.Type, userprop.Value);
            
            else
                throw new SerialisationException($"Invalid type; Target DataType is {type} and value DataType is {value?.GetType().ToString() ?? "null"}");
            
        }

        public static void WriteBinaryValue<T>(this CoreBinaryWriter writer, T value)
            => WriteBinaryValue(writer, typeof(T), value);


        /// <summary>
        /// Binary deserialize a bunch of different types of values. <see cref="WriteBinaryValue(CoreBinaryWriter, Type, object?)"/> and
        /// <see cref="ReadBinaryValue(CoreBinaryReader, Type)"/> are inverses of each other.
        /// </summary>
        /// <remarks>
        /// Handles <see cref="byte"/>[], <see cref="Array"/>s of serialisable values, <see cref="Enum"/>, <see cref="bool"/>, <see cref="string"/>,
        /// <see cref="Guid"/>, <see cref="byte"/>, <see cref="Int16"/>, <see cref="Int32"/>, <see cref="Int64"/>, <see cref="float"/>, <see cref="double"/>,
        /// <see cref="DateTime"/>, <see cref="TimeSpan"/>, <see cref="LoggablePropertyAttribute"/>, <see cref="IPackable"/>, <see cref="Nullable{T}"/>
        /// and <see cref="ISerializeBinary"/>.
        /// </remarks>
        /// <param name="reader"></param>
        /// <param name="type"></param>
        /// <exception cref="Exception">If an object of <paramref name="type"/> is unable to be deserialized.</exception>
        public static object? ReadBinaryValue(this CoreBinaryReader reader, Type type)
        {
            if (type == typeof(byte[]))
            {
                var length = reader.ReadInt32();
                return reader.ReadBytes(length);
            }
            else if (type.IsArray)
            {
                var length = reader.ReadInt32();
                var elementType = type.GetElementType();

                var array = Array.CreateInstance(elementType, length);
                for (int i = 0; i < array.Length; ++i)
                {
                    array.SetValue(ReadBinaryValue(reader, elementType), i);
                }
                return array;
            }
            else if (type.IsEnum)
            {
                var val = ReadBinaryValue(reader, type.GetEnumUnderlyingType());
                return Enum.ToObject(type, val);
            }
            else if (type == typeof(bool))
            {
                return reader.ReadBoolean();
            }
            else if (type == typeof(string))
            {
                return reader.ReadString();
            }
            else if (type == typeof(Guid))
            {
                return reader.ReadGuid();
            }
            else if (type == typeof(byte))
            {
                return reader.ReadByte();
            }
            else if (type == typeof(Int16))
            {
                return reader.ReadInt16();
            }
            else if (type == typeof(Int32))
            {
                return reader.ReadInt32();
            }
            else if (type == typeof(Int64))
            {
                return reader.ReadInt64();
            }
            else if (type == typeof(float))
            {
                return reader.ReadSingle();
            }
            else if (type == typeof(double))
            {
                return reader.ReadDouble();
            }
            else if (type == typeof(DateTime))
            {
                return new DateTime(reader.ReadInt64());
            }
            else if (type == typeof(TimeSpan))
            {
                return new TimeSpan(reader.ReadInt64());
            }
            else if (type == typeof(LoggablePropertyAttribute))
            {
                String format = reader.ReadString();
                return String.IsNullOrWhiteSpace(format)
                    ? null
                    : new LoggablePropertyAttribute() { Format = format };
            }
            else if (typeof(IPackable).IsAssignableFrom(type))
            {
                if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean.
                {
                    var packable = (Activator.CreateInstance(type) as IPackable)!;
                    packable.Unpack(reader);
                    return packable;
                }
                else
                {
                    return null;
                }
            }
            else if (typeof(ISerializeBinary).IsAssignableFrom(type))
            {
                if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean.
                {
                    var obj = (Activator.CreateInstance(type, true) as ISerializeBinary)!;
                    obj.DeserializeBinary(reader);
                    return obj;
                }
                else
                {
                    return null;
                }
            }
            else if (Nullable.GetUnderlyingType(type) is Type t)
            {
                var isNull = reader.ReadBoolean();
                if (isNull)
                {
                    return null;
                }
                else
                {
                    return reader.ReadBinaryValue(t);
                }
            }
            else
            {
                throw new SerialisationException($"Invalid type; Target DataType is {type}");
            }
        }

        public static T ReadBinaryValue<T>(this CoreBinaryReader reader)
        {
            var result = ReadBinaryValue(reader, typeof(T));
            return (result != null ? (T)result : default)!;
        }

        public static IEnumerable<IProperty> SerializableProperties(Type type, Predicate<IProperty>? filter = null) =>
            DatabaseSchema.Properties(type)
                .Where(x => (!(x is StandardProperty st) || st.IsSerializable) && (filter?.Invoke(x) ?? true));

        private static void WriteOriginalValues<TObject>(this CoreBinaryWriter writer, TObject obj)
            where TObject : BaseObject
        {
            var originalValues = new List<Tuple<Type, string, object?>>();
            foreach (var (key, value) in obj.OriginalValueList)
            {
                if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop && prop.IsSerializable)
                {
                    originalValues.Add(new Tuple<Type, string, object?>(prop.PropertyType, key, value));
                }
            }

            writer.Write(originalValues.Count);
            foreach (var (type, key, value) in originalValues)
            {
                writer.Write(key);
                try
                {
                    writer.WriteBinaryValue(type, value);
                }
                catch (Exception e)
                {
                    CoreUtils.LogException("", e, "Error serialising OriginalValues");
                }
            }
        }
        private static void ReadOriginalValues<TObject>(this CoreBinaryReader reader, TObject obj)
            where TObject : BaseObject
        {
            var nOriginalValues = reader.ReadInt32();
            for (int i = 0; i < nOriginalValues; ++i)
            {
                var key = reader.ReadString();
                if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop)
                {
                    var value = reader.ReadBinaryValue(prop.PropertyType);
                    obj.OriginalValueList[prop.Name] = value;
                }
            }
        }
        public static void WriteObject<TObject>(this CoreBinaryWriter writer, TObject entity, Type type)
            where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var properties = SerializableProperties(type).ToList();

            writer.Write(properties.Count);
            foreach (var property in properties)
            {
                writer.Write(property.Name);
                writer.WriteBinaryValue(property.PropertyType, property.Getter()(entity));
            }
            writer.WriteOriginalValues(entity);
        }

        /// <summary>
        /// An implementation of binary serialising a <typeparamref name="TObject"/>; this is the inverse of <see cref="ReadObject{TObject}(CoreBinaryReader)"/>.
        /// </summary>
        /// <remarks>
        /// Also serialises the names of properties along with the values.
        /// </remarks>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="writer"></param>
        /// <param name="entity"></param>
        public static void WriteObject<TObject>(this CoreBinaryWriter writer, TObject entity)
            where TObject : BaseObject, new() => WriteObject(writer, entity, typeof(TObject));

        public static TObject ReadObject<TObject>(this CoreBinaryReader reader, Type type)
            where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var obj = (Activator.CreateInstance(type) as TObject)!;
            obj.SetObserving(false);

            var nProps = reader.ReadInt32();
            for (int i = 0; i < nProps; ++i)
            {
                var propName = reader.ReadString();
                var property = DatabaseSchema.Property(type, propName)
                    ?? throw new SerialisationException($"Property {propName} does not exist on {type.EntityName()}");
                property.Setter()(obj, reader.ReadBinaryValue(property.PropertyType));
            }
            reader.ReadOriginalValues(obj);

            obj.SetObserving(true);

            return obj;
        }

        /// <summary>
        /// The inverse of <see cref="WriteObject{TObject}(CoreBinaryWriter, TObject)"/>.
        /// </summary>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="reader"></param>
        /// <returns></returns>
        public static TObject ReadObject<TObject>(this CoreBinaryReader reader)
            where TObject : BaseObject, new() => reader.ReadObject<TObject>(typeof(TObject));

        /// <summary>
        /// An implementation of binary serialising multiple <typeparamref name="TObject"/>s;
        /// this is the inverse of <see cref="ReadObjects{TObject}(CoreBinaryReader)"/>.
        /// </summary>
        /// <remarks>
        /// Also serialises the names of properties along with the values.
        /// </remarks>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="writer"></param>
        /// <param name="objects"></param>
        public static void WriteObjects<TObject>(this CoreBinaryWriter writer, ICollection<TObject>? objects)
            where TObject : BaseObject, new() => WriteObjects(writer, typeof(TObject), objects);

        public static void WriteObjects<TObject>(this CoreBinaryWriter writer, Type type, ICollection<TObject>? objects, Predicate<IProperty>? filter = null)
            where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var nObjs = objects?.Count ?? 0;
            writer.Write(nObjs);

            if (nObjs == 0)
            {
                return;
            }

            var properties = SerializableProperties(type, filter).ToList();
            writer.Write(properties.Count);
            foreach (var property in properties)
            {
                writer.Write(property.Name);
            }

            if(objects != null)
            {
                foreach (var obj in objects)
                {
                    foreach (var property in properties)
                    {
                        writer.WriteBinaryValue(property.PropertyType, property.Getter()(obj));
                    }
                    writer.WriteOriginalValues(obj);
                }
            }
        }

        /// <summary>
        /// The inverse of <see cref="WriteObjects{TObject}(CoreBinaryWriter, ICollection{TObject})"/>.
        /// </summary>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="reader"></param>
        /// <returns></returns>
        public static List<TObject> ReadObjects<TObject>(this CoreBinaryReader reader)
            where TObject : BaseObject, new()
        {
            return ReadObjects<TObject>(reader, typeof(TObject));
        }
        
        public static List<TObject> ReadObjects<TObject>(this CoreBinaryReader reader, Type type) where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");
            
            var objs = new List<TObject>();
            var properties = new List<IProperty>();

            var nObjs = reader.ReadInt32();
            if(nObjs == 0)
            {
                return objs;
            }

            var nProps = reader.ReadInt32();
            for (int i = 0; i < nProps; ++i)
            {
                var propertyName = reader.ReadString();
                var property = DatabaseSchema.Property(type, propertyName)
                    ?? throw new SerialisationException($"Property {propertyName} does not exist on {type.EntityName()}"); 
                properties.Add(property);
            }

            for (int i = 0; i < nObjs; ++i)
            {
                var obj = (Activator.CreateInstance(type) as TObject)!;
                obj.SetObserving(false);

                foreach (var property in properties)
                {
                    property.Setter()(obj, reader.ReadBinaryValue(property.PropertyType));
                }
                reader.ReadOriginalValues(obj);

                obj.SetObserving(true);

                objs.Add(obj);
            }
            return objs;
        }
    }

    public abstract class CustomJsonConverter<T> : JsonConverter<T>
    {
        protected object? ReadJson(ref Utf8JsonReader reader)
        {
            switch (reader.TokenType)
            {
                case JsonTokenType.String:
                    return reader.GetString();
                case JsonTokenType.Number:
                    if (reader.TryGetInt32(out int intValue))
                        return intValue;
                    if (reader.TryGetDouble(out double doubleValue))
                        return doubleValue;
                    return null;
                case JsonTokenType.True:
                    return true;
                case JsonTokenType.False:
                    return false;
                case JsonTokenType.Null:
                    return null;
                case JsonTokenType.StartArray:
                    var values = new List<object?>();
                    reader.Read();
                    while(reader.TokenType != JsonTokenType.EndArray)
                    {
                        values.Add(ReadJson(ref reader));
                        reader.Read();
                    }
                    return values;
                default:
                    return null;
            }
        }
        
        /// <summary>
        /// Write a value as a JSON object; note that some data types, like
        /// <see cref="Guid"/> and <see cref="DateTime"/> will be encoded as
        /// strings, and therefore will be returned as strings when read by
        /// <see cref="ReadJson(Utf8JsonReader)"/>. However, all types that
        /// this can write should be able to be retrieved by calling <see
        /// cref="CoreUtils.ChangeType(object?, Type)"/> on the resultant
        /// value.
        /// </summary>
        protected void WriteJson(Utf8JsonWriter writer, object? value)
        {
            if  (value == null)
                writer.WriteNullValue();
            else if (value is string sVal)
                writer.WriteStringValue(sVal);
            else if (value is bool bVal)
                writer.WriteBooleanValue(bVal);
            else if (value is byte b)
                writer.WriteNumberValue(b);
            else if (value is short i16)
                writer.WriteNumberValue(i16);
            else if (value is int i32)
                writer.WriteNumberValue(i32);
            else if (value is long i64)
                writer.WriteNumberValue(i64);
            else if (value is float f)
                writer.WriteNumberValue(f);
            else if (value is double dVal)
                writer.WriteNumberValue(dVal);
            else if (value is DateTime dtVal)
                writer.WriteStringValue(dtVal.ToString());
            else if (value is TimeSpan tsVal)
                writer.WriteStringValue(tsVal.ToString());
            else if (value is Guid guid)
                writer.WriteStringValue(guid.ToString());
            else if(value is byte[] arr)
            {
                writer.WriteBase64StringValue(arr);
            }
            else if(value is Array array)
            {
                writer.WriteStartArray();
                foreach(var val1 in array)
                {
                    WriteJson(writer, val1);
                }
                writer.WriteEndArray();
            }
            else if(value is Enum e)
            {
                WriteJson(writer, Convert.ChangeType(e, e.GetType().GetEnumUnderlyingType()));
            }
            else
            {
                Logger.Send(LogType.Error, "", $"Could not write object of type {value.GetType()} as JSON");
            }
        }
        
        protected void WriteJson(Utf8JsonWriter writer, string name, object? value)
        {
            writer.WritePropertyName(name);
            WriteJson(writer, value);
        }
    }

    public class BaseObjectJSONConverter : CustomJsonConverter<BaseObject>
    {
        public override bool CanConvert(Type objectType)
        {
            return objectType.IsSubclassOf(typeof(BaseObject));
        }

        public override BaseObject? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
        {
            BaseObject obj = (Activator.CreateInstance(typeToConvert) as BaseObject)!;
            obj.OriginalValues ??= new ConcurrentDictionary<string, object?>();
            obj.SetObserving(false);

            if (reader.TokenType == JsonTokenType.StartObject)
            {
                reader.Read();
                while (reader.TokenType != JsonTokenType.EndObject)
                {
                    if (reader.TokenType != JsonTokenType.PropertyName)
                        throw new JsonException("Expected PropertyName token.");

                    string propertyName = reader.GetString();
                    reader.Read(); // Advance to the property value

                    if (Equals(propertyName, "OriginalValues"))
                    {
                        while (reader.Read())
                        {
                            if (reader.TokenType == JsonTokenType.EndObject)
                                break;
                            string? name = reader.GetString();
                            reader.Read();
                            if (!string.IsNullOrWhiteSpace(name))
                                obj.OriginalValues[name] = ReadJson(ref reader);

                        }
                    }
                    else if (DatabaseSchema.Property(typeToConvert, propertyName) is IProperty prop)
                    {
                        var value = ReadJson(ref reader);
                        prop.Setter()(obj, value);
                    }
                }
            }
            obj.SetObserving(true);
            return obj;
        }

        

        public override void Write(Utf8JsonWriter writer, BaseObject obj, JsonSerializerOptions options)
        {
            writer.WriteStartObject();
            writer.WritePropertyName("OriginalValues");
            writer.WriteStartObject();
            if (obj.OriginalValues != null)
            {
                foreach (var key in obj.OriginalValues.Keys)
                {
                    var val = obj.OriginalValues[key];
                    if (val == null)
                        writer.WriteNull(key);
                    else
                        writer.WriteString(key, val.ToString());
                }
            }
            foreach(var property in DatabaseSchema.Properties(obj.GetType()))
            {
                var val = property.Getter()(obj);
                WriteJson(writer, property.Name, property.Getter());
            }
            writer.WriteEndObject();
            
        }
        
    }
}