inabox/InABox.Core/Serialization.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json.Serialization.Metadata;
using InABox.Clients;

namespace InABox.Core
{
    public enum SerializationFormat
    {
        Json,
        Binary
    }

    public class SerialisationException : Exception
    {
        public SerialisationException(string message) : base(message) { }
    }

    public interface ISerializeBinary
    {
        public void SerializeBinary(CoreBinaryWriter writer);

        public void DeserializeBinary(CoreBinaryReader reader);
    }

    public static class Serialization
    {
        /// <summary>
        /// TypeInfoResolver modifier that removes properties that don't have setters.
        /// </summary>
        /// <param name="typeInfo"></param>
        public static void WritablePropertiesOnly(JsonTypeInfo typeInfo)
        {
            if (typeInfo.Kind == JsonTypeInfoKind.Object)
            {
                var toRemove = typeInfo.Properties.Where(x => x.Set is null).ToList();
                foreach (var prop in toRemove)
                {
                    typeInfo.Properties.Remove(prop);
                }
            }
        }

        public static List<JsonConverter> DefaultConverters { get; } = new List<JsonConverter>()
        {
            new CoreTableJsonConverter(),
            new FilterJsonConverter(),
            new ColumnJsonConverter(),
            new ColumnsJsonConverter(),
            new SortOrderJsonConverter(),
            new MultiQueryRequestConverter(),
            new UserPropertiesJsonConverter(),
            new TypeJsonConverter(),
            new PolymorphicConverter(),
            new ObjectConverter(), // Our fallback, which converts JSON objects into real ones.
        };

        private static JsonSerializerOptions SerializerSettings(bool indented = true, bool populateObject = false)
        {
            return CreateSerializerSettings(indented, populateObject);
        }

        public static JsonSerializerOptions CreateSerializerSettings(bool indented = true, bool populateObject = false)
        {
            var settings = new JsonSerializerOptions { };

            foreach (var converter in DefaultConverters)
            {
                settings.Converters.Add(converter);
            }
            if (populateObject)
            {
                settings.TypeInfoResolver = new PopulateTypeInfoResolver(new DefaultJsonTypeInfoResolver());
            }

            settings.WriteIndented = indented;
            return settings;
        }

        public static string Serialize(object? o, bool indented = false)
        {
            var json = JsonSerializer.Serialize(o, SerializerSettings(indented));
            return json;
        }

        public static void Serialize(object o, Stream stream, bool indented = false)
        {
            var settings = SerializerSettings(indented);
            JsonSerializer.Serialize(stream, o, settings);
        }

        [return: MaybeNull]
        public static T Deserialize<T>(Stream? stream, bool strict = false)
        {
            if (stream == null)
                return default;

            try
            {
                var settings = SerializerSettings();
                return JsonSerializer.Deserialize<T>(stream, settings);
            }
            catch (Exception e)
            {
                if (strict)
                    throw;

                Logger.Send(LogType.Error, ClientFactory.UserID, $"Error in Deserialize<{typeof(T)}>(): {e.Message}");
                return default;
            }
        }

        public static object? Deserialize(Type type, Stream? stream)
        {
            if (stream == null)
                return null;

            object? result = null;
            var settings = SerializerSettings();
            result = JsonSerializer.Deserialize(stream, type, settings);

            return result;
        }

        [return: MaybeNull]
        public static T Deserialize<T>(string? json, bool strict = false) // where T : new()
        {
            var ret = default(T);
            if (string.IsNullOrWhiteSpace(json))
                return ret;

            try
            {
                var settings = SerializerSettings();
                if (typeof(T).IsArray)
                {
                    ret = JsonSerializer.Deserialize<T>(json, settings);
                }
                else
                {
                    ret = JsonSerializer.Deserialize<T>(json, settings);
                }
            }
            catch (Exception e)
            {
                if (strict)
                {
                    throw;
                }

                CoreUtils.LogException("", e);
                if (typeof(T).IsArray)
                {
                    ret = (T)(object)Array.CreateInstance(typeof(T).GetElementType(), 0);
                }
                else
                {
                    ret = (T)Activator.CreateInstance(typeof(T), true);
                }
            }

            return ret;
        }

        [return: MaybeNull]
        public static void DeserializeInto<T>(string? json, T obj, bool strict = false)
        {
            if (string.IsNullOrWhiteSpace(json))
                return;

            try
            {
                var settings = SerializerSettings(populateObject: true);
                PopulateTypeInfoResolver.t_populateObject = obj;
                if (typeof(T).IsArray)
                {
                    JsonSerializer.Deserialize<T>(json, settings);
                }
                else
                {
                    JsonSerializer.Deserialize<T>(json, settings);
                }
            }
            catch (Exception e)
            {
                if (strict)
                {
                    throw;
                }

                CoreUtils.LogException("", e);
            }
            finally
            {
                PopulateTypeInfoResolver.t_populateObject = null;
            }
        }

        public static object? Deserialize(Type T, string json) // where T : new()
        {
            var ret = T.GetDefault();
            if (string.IsNullOrWhiteSpace(json))
                return ret;

            try
            {
                var settings = SerializerSettings();
                if (T.IsArray)
                {
                    object o = Array.CreateInstance(T.GetElementType(), 0);
                    ret = o;
                }
                else
                {
                    ret = JsonSerializer.Deserialize(json, T, settings);
                }
            }
            catch (Exception)
            {
                ret = Activator.CreateInstance(T, true);
            }

            return ret;
        }

        #region Binary Serialization

        public static byte[] WriteBinary(this ISerializeBinary obj, BinarySerializationSettings settings)
        {
            using var stream = new MemoryStream();
            obj.SerializeBinary(new CoreBinaryWriter(stream, settings));

            return stream.ToArray();
        }
        public static void WriteBinary(this ISerializeBinary obj, Stream stream, BinarySerializationSettings settings)
        {
            obj.SerializeBinary(new CoreBinaryWriter(stream, settings));
        }

        public static T ReadBinary<T>(byte[] data, BinarySerializationSettings settings)
            where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), data, settings);
        public static T ReadBinary<T>(Stream stream, BinarySerializationSettings settings)
            where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), stream, settings);

        public static object ReadBinary(Type T, byte[] data, BinarySerializationSettings settings)
        {
            using var stream = new MemoryStream(data);
            return ReadBinary(T, stream, settings);
        }
        public static object ReadBinary(Type T, Stream stream, BinarySerializationSettings settings)
        {
            var obj = (Activator.CreateInstance(T) as ISerializeBinary)!;
            obj.DeserializeBinary(new CoreBinaryReader(stream, settings));
            return obj;
        }

        #endregion
    }

    internal class PopulateTypeInfoResolver : IJsonTypeInfoResolver
    {
        private readonly IJsonTypeInfoResolver? _jsonTypeInfoResolver;

        [ThreadStatic]
        internal static object? t_populateObject;

        public PopulateTypeInfoResolver(IJsonTypeInfoResolver? jsonTypeInfoResolver)
        {
            _jsonTypeInfoResolver = jsonTypeInfoResolver;
        }

        public JsonTypeInfo? GetTypeInfo(Type type, JsonSerializerOptions options)
        {
            var typeInfo = _jsonTypeInfoResolver?.GetTypeInfo(type, options);
            if (typeInfo != null && typeInfo.Kind != JsonTypeInfoKind.None)
            {
                var defaultCreateObject = typeInfo.CreateObject;
                if (defaultCreateObject != null)
                {
                    typeInfo.CreateObject = () =>
                    {
                        if (t_populateObject != null)
                        {
                            var result = t_populateObject;
                            t_populateObject = null;
                            return result;
                        }
                        else
                        {
                            return defaultCreateObject.Invoke();
                        }
                    };
                }
            }
            return typeInfo;
        }
    }

    public class CoreBinaryReader : BinaryReader
    {
        public BinarySerializationSettings Settings { get; set; }

        public CoreBinaryReader(Stream stream, BinarySerializationSettings settings) : base(stream)
        {
            Settings = settings;
        }
    }
    public class CoreBinaryWriter : BinaryWriter
    {
        public BinarySerializationSettings Settings { get; set; }

        public CoreBinaryWriter(Stream stream, BinarySerializationSettings settings) : base(stream)
        {
            Settings = settings;
        }
    }

    /// <summary>
    /// A class to maintain the consistency of serialisation formats across versions.
    /// The design of this is such that specific versions of serialisation have different parameters set,
    /// and the versions are maintained as static properties. Please keep the constructor private.
    /// </summary>
    /// <remarks>
    /// Note that <see cref="Latest"/> should always be updated to point to the latest version.
    /// <br/>
    /// Note also that all versions should have an entry in the <see cref="ConvertVersionString(string)"/> function.
    /// <br/>
    /// Also, if you create a new format, it would probably be a good idea to add a database update script to get all
    /// <see cref="IPackable"/> and <see cref="ISerializeBinary"/> properties and update the version of the format.
    /// (Otherwise, we'd basically be nullifying all data that is currently binary serialised.)
    /// </remarks>
    public class BinarySerializationSettings
    {

        /// <summary>
        /// Should the Info() call return RPC and Rest Ports?  This is
        /// To workaround a bug in RPCsockets that crash on large uploads
        /// </summary>
        /// <remarks>
        /// True in all serialization versions >= 1.2
        /// </remarks>
        public bool RPCClientWorkaround { get; set; }

        /// <summary>
        /// Should reference types include a flag for nullability? (Adds an extra boolean field for whether the value is null or not).
        /// </summary>
        /// <remarks>
        /// True in all serialisation versions >= 1.1.
        /// </remarks>
        public bool IncludeNullables { get; set; }

        public string Version { get; set; }

        public static BinarySerializationSettings Latest => V1_2;



        public static BinarySerializationSettings V1_0 = new BinarySerializationSettings("1.0")
        {
            IncludeNullables = false,
            RPCClientWorkaround = false
        };

        public static BinarySerializationSettings V1_1 = new BinarySerializationSettings("1.1")
        {
            IncludeNullables = true,
            RPCClientWorkaround = false
        };

        public static BinarySerializationSettings V1_2 = new BinarySerializationSettings("1.2")
        {
            IncludeNullables = true,
            RPCClientWorkaround = true
        };

        public static BinarySerializationSettings ConvertVersionString(string version) => version switch
        {
            "1.0" => V1_0,
            "1.1" => V1_1,
            "1.2" => V1_2,
            _ => V1_0
        };

        private BinarySerializationSettings(string version)
        {
            Version = version;
        }
    }

    public static class SerializationUtils
    {
        public static void Write(this BinaryWriter writer, Guid guid)
        {
            writer.Write(guid.ToByteArray());
        }
        public static Guid ReadGuid(this BinaryReader reader)
        {
            return new Guid(reader.ReadBytes(16));
        }

        public static void Write(this BinaryWriter writer, DateTime dateTime)
        {
            writer.Write(dateTime.Ticks);
        }
        public static DateTime ReadDateTime(this BinaryReader reader)
        {
            return new DateTime(reader.ReadInt64());
        }

        private static bool MatchType<T1>(Type t) => typeof(T1) == t;
        private static bool MatchType<T1, T2>(Type t) => (typeof(T1) == t) || (typeof(T2) == t);

        /// <summary>
        /// Binary serialize a bunch of different types of values. <see cref="WriteBinaryValue(CoreBinaryWriter, Type, object?)"/> and
        /// <see cref="ReadBinaryValue(CoreBinaryReader, Type)"/> are inverses of each other.
        /// </summary>
        /// <remarks>
        /// Handles <see cref="byte"/>[], <see cref="Array"/>s of serialisable values, <see cref="Enum"/>, <see cref="bool"/>, <see cref="string"/>,
        /// <see cref="Guid"/>, <see cref="byte"/>, <see cref="Int16"/>, <see cref="Int32"/>, <see cref="Int64"/>, <see cref="float"/>, <see cref="double"/>,
        /// <see cref="DateTime"/>, <see cref="TimeSpan"/>, <see cref="LoggablePropertyAttribute"/>, <see cref="IPackable"/>, <see cref="Nullable{T}"/>
        /// and <see cref="ISerializeBinary"/>.
        /// </remarks>
        /// <param name="writer"></param>
        /// <param name="type"></param>
        /// <param name="value"></param>
        /// <exception cref="Exception">If an object of <paramref name="type"/> is unable to be serialized.</exception>
        public static void WriteBinaryValue(this CoreBinaryWriter writer, Type type, object? value)
        {
            value ??= CoreUtils.GetDefault(type);

            if (value == null)
            {
                if (MatchType<string>(type))
                    writer.Write("");
                else if (writer.Settings.IncludeNullables && typeof(IPackable).IsAssignableFrom(type))
                    writer.Write(false);
                else if (writer.Settings.IncludeNullables && typeof(ISerializeBinary).IsAssignableFrom(type))
                    writer.Write(false);
                else if (Nullable.GetUnderlyingType(type) is Type t)
                    writer.Write(false);
                else if (MatchType<LoggablePropertyAttribute, object>(type))
                    writer.Write("");
                else
                    writer.Write(0);
            }

            else if (MatchType<byte[], object>(type) && value is byte[] bArray)
            {
                writer.Write(bArray.Length);
                writer.Write(bArray);
            }

            else if (type.IsArray && value is Array array)
            {
                var elementType = type.GetElementType();
                writer.Write(array.Length);
                foreach (var val1 in array)
                {
                    WriteBinaryValue(writer, elementType, val1);
                }
            }

            else if (type.IsEnum && value is Enum e)
            {
                var underlyingType = type.GetEnumUnderlyingType();
                WriteBinaryValue(writer, underlyingType, Convert.ChangeType(e, underlyingType));
            }

            else if (MatchType<bool, object>(type) && value is bool b)
            {
                writer.Write(b);
            }

            else if (MatchType<string, object>(type) && value is string str)
                writer.Write(str);

            else if (MatchType<Guid, object>(type) && value is Guid guid)
                writer.Write(guid);

            else if (MatchType<byte, object>(type) && value is byte i8)
                writer.Write(i8);

            else if (MatchType<Int16, object>(type) && value is Int16 i16)
                writer.Write(i16);

            else if (MatchType<Int32, object>(type) && value is Int32 i32)
                writer.Write(i32);

            else if (MatchType<Int64, object>(type) && value is Int64 i64)
                writer.Write(i64);

            else if (MatchType<float, object>(type) && value is float f32)
                writer.Write(f32);

            else if (MatchType<double, object>(type) && value is double f64)
                writer.Write(f64);

            else if (MatchType<DateTime, object>(type) && value is DateTime date)
                writer.Write(date.Ticks);

            else if (MatchType<TimeSpan, object>(type) && value is TimeSpan time)
                writer.Write(time.Ticks);

            else if (MatchType<LoggablePropertyAttribute, object>(type) && value is LoggablePropertyAttribute lpa)
                writer.Write(lpa.Format ?? string.Empty);

            else if (typeof(IPackable).IsAssignableFrom(type) && value is IPackable pack)
            {
                if (writer.Settings.IncludeNullables)
                    writer.Write(true);
                pack.Pack(writer);
            }

            else if (typeof(ISerializeBinary).IsAssignableFrom(type) && value is ISerializeBinary binary)
            {
                if (writer.Settings.IncludeNullables)
                    writer.Write(true);
                binary.SerializeBinary(writer);
            }

            else if (Nullable.GetUnderlyingType(type) is Type t)
            {
                writer.Write(true);
                writer.WriteBinaryValue(t, value);
            }

            else if (value is UserProperty userprop)
                WriteBinaryValue(writer, userprop.Type, userprop.Value);

            else
                throw new SerialisationException($"Invalid type; Target DataType is {type} and value DataType is {value?.GetType().ToString() ?? "null"}");

        }

        public static void WriteBinaryValue<T>(this CoreBinaryWriter writer, T value)
            => WriteBinaryValue(writer, typeof(T), value);


        /// <summary>
        /// Binary deserialize a bunch of different types of values. <see cref="WriteBinaryValue(CoreBinaryWriter, Type, object?)"/> and
        /// <see cref="ReadBinaryValue(CoreBinaryReader, Type)"/> are inverses of each other.
        /// </summary>
        /// <remarks>
        /// Handles <see cref="byte"/>[], <see cref="Array"/>s of serialisable values, <see cref="Enum"/>, <see cref="bool"/>, <see cref="string"/>,
        /// <see cref="Guid"/>, <see cref="byte"/>, <see cref="Int16"/>, <see cref="Int32"/>, <see cref="Int64"/>, <see cref="float"/>, <see cref="double"/>,
        /// <see cref="DateTime"/>, <see cref="TimeSpan"/>, <see cref="LoggablePropertyAttribute"/>, <see cref="IPackable"/>, <see cref="Nullable{T}"/>
        /// and <see cref="ISerializeBinary"/>.
        /// </remarks>
        /// <param name="reader"></param>
        /// <param name="type"></param>
        /// <exception cref="Exception">If an object of <paramref name="type"/> is unable to be deserialized.</exception>
        public static object? ReadBinaryValue(this CoreBinaryReader reader, Type type)
        {
            if (type == typeof(byte[]))
            {
                var length = reader.ReadInt32();
                return reader.ReadBytes(length);
            }
            else if (type.IsArray)
            {
                var length = reader.ReadInt32();
                var elementType = type.GetElementType();

                var array = Array.CreateInstance(elementType, length);
                for (int i = 0; i < array.Length; ++i)
                {
                    array.SetValue(ReadBinaryValue(reader, elementType), i);
                }
                return array;
            }
            else if (type.IsEnum)
            {
                var val = ReadBinaryValue(reader, type.GetEnumUnderlyingType());
                return Enum.ToObject(type, val);
            }
            else if (type == typeof(bool))
            {
                return reader.ReadBoolean();
            }
            else if (type == typeof(string))
            {
                return reader.ReadString();
            }
            else if (type == typeof(Guid))
            {
                return reader.ReadGuid();
            }
            else if (type == typeof(byte))
            {
                return reader.ReadByte();
            }
            else if (type == typeof(Int16))
            {
                return reader.ReadInt16();
            }
            else if (type == typeof(Int32))
            {
                return reader.ReadInt32();
            }
            else if (type == typeof(Int64))
            {
                return reader.ReadInt64();
            }
            else if (type == typeof(float))
            {
                return reader.ReadSingle();
            }
            else if (type == typeof(double))
            {
                return reader.ReadDouble();
            }
            else if (type == typeof(DateTime))
            {
                return new DateTime(reader.ReadInt64());
            }
            else if (type == typeof(TimeSpan))
            {
                return new TimeSpan(reader.ReadInt64());
            }
            else if (type == typeof(LoggablePropertyAttribute))
            {
                String format = reader.ReadString();
                return String.IsNullOrWhiteSpace(format)
                    ? null
                    : new LoggablePropertyAttribute() { Format = format };
            }
            else if (typeof(IPackable).IsAssignableFrom(type))
            {
                if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean.
                {
                    var packable = (Activator.CreateInstance(type) as IPackable)!;
                    packable.Unpack(reader);
                    return packable;
                }
                else
                {
                    return null;
                }
            }
            else if (typeof(ISerializeBinary).IsAssignableFrom(type))
            {
                if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean.
                {
                    var obj = (Activator.CreateInstance(type, true) as ISerializeBinary)!;
                    obj.DeserializeBinary(reader);
                    return obj;
                }
                else
                {
                    return null;
                }
            }
            else if (Nullable.GetUnderlyingType(type) is Type t)
            {
                var isNull = reader.ReadBoolean();
                if (isNull)
                {
                    return null;
                }
                else
                {
                    return reader.ReadBinaryValue(t);
                }
            }
            else
            {
                throw new SerialisationException($"Invalid type; Target DataType is {type}");
            }
        }

        public static T ReadBinaryValue<T>(this CoreBinaryReader reader)
        {
            var result = ReadBinaryValue(reader, typeof(T));
            return (result != null ? (T)result : default)!;
        }

        public static IEnumerable<IProperty> SerializableProperties(Type type, Predicate<IProperty>? filter = null) =>
            DatabaseSchema.Properties(type)
                .Where(x => (!(x is StandardProperty st) || st.IsSerializable) && (filter?.Invoke(x) ?? true));

        private static void WriteOriginalValues<TObject>(this CoreBinaryWriter writer, TObject obj)
            where TObject : BaseObject
        {
            var originalValues = new List<Tuple<Type, string, object?>>();
            foreach (var (key, value) in obj.OriginalValueList)
            {
                if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop && prop.IsSerializable)
                {
                    originalValues.Add(new Tuple<Type, string, object?>(prop.PropertyType, key, value));
                }
            }

            writer.Write(originalValues.Count);
            foreach (var (type, key, value) in originalValues)
            {
                writer.Write(key);
                try
                {
                    writer.WriteBinaryValue(type, value);
                }
                catch (Exception e)
                {
                    CoreUtils.LogException("", e, "Error serialising OriginalValues");
                }
            }
        }
        private static void ReadOriginalValues<TObject>(this CoreBinaryReader reader, TObject obj)
            where TObject : BaseObject
        {
            var nOriginalValues = reader.ReadInt32();
            for (int i = 0; i < nOriginalValues; ++i)
            {
                var key = reader.ReadString();
                if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop)
                {
                    var value = reader.ReadBinaryValue(prop.PropertyType);
                    obj.OriginalValueList[prop.Name] = value;
                }
            }
        }
        public static void WriteObject<TObject>(this CoreBinaryWriter writer, TObject entity, Type type)
            where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var properties = SerializableProperties(type).ToList();

            writer.Write(properties.Count);
            foreach (var property in properties)
            {
                writer.Write(property.Name);
                writer.WriteBinaryValue(property.PropertyType, property.Getter()(entity));
            }
            writer.WriteOriginalValues(entity);
        }

        /// <summary>
        /// An implementation of binary serialising a <typeparamref name="TObject"/>; this is the inverse of <see cref="ReadObject{TObject}(CoreBinaryReader)"/>.
        /// </summary>
        /// <remarks>
        /// Also serialises the names of properties along with the values.
        /// </remarks>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="writer"></param>
        /// <param name="entity"></param>
        public static void WriteObject<TObject>(this CoreBinaryWriter writer, TObject entity)
            where TObject : BaseObject, new() => WriteObject(writer, entity, typeof(TObject));

        public static TObject ReadObject<TObject>(this CoreBinaryReader reader, Type type)
            where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var obj = (Activator.CreateInstance(type) as TObject)!;
            obj.SetObserving(false);

            var nProps = reader.ReadInt32();
            for (int i = 0; i < nProps; ++i)
            {
                var propName = reader.ReadString();
                var property = DatabaseSchema.Property(type, propName)
                    ?? throw new SerialisationException($"Property {propName} does not exist on {type.EntityName()}");
                property.Setter()(obj, reader.ReadBinaryValue(property.PropertyType));
            }
            reader.ReadOriginalValues(obj);

            obj.SetObserving(true);

            return obj;
        }

        /// <summary>
        /// The inverse of <see cref="WriteObject{TObject}(CoreBinaryWriter, TObject)"/>.
        /// </summary>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="reader"></param>
        /// <returns></returns>
        public static TObject ReadObject<TObject>(this CoreBinaryReader reader)
            where TObject : BaseObject, new() => reader.ReadObject<TObject>(typeof(TObject));

        /// <summary>
        /// An implementation of binary serialising multiple <typeparamref name="TObject"/>s;
        /// this is the inverse of <see cref="ReadObjects{TObject}(CoreBinaryReader)"/>.
        /// </summary>
        /// <remarks>
        /// Also serialises the names of properties along with the values.
        /// </remarks>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="writer"></param>
        /// <param name="objects"></param>
        public static void WriteObjects<TObject>(this CoreBinaryWriter writer, ICollection<TObject>? objects)
            where TObject : BaseObject, new() => WriteObjects(writer, typeof(TObject), objects);

        public static void WriteObjects<TObject>(this CoreBinaryWriter writer, Type type, ICollection<TObject>? objects, Predicate<IProperty>? filter = null)
            where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var nObjs = objects?.Count ?? 0;
            writer.Write(nObjs);

            if (nObjs == 0)
            {
                return;
            }

            var properties = SerializableProperties(type, filter).ToList();
            writer.Write(properties.Count);
            foreach (var property in properties)
            {
                writer.Write(property.Name);
            }

            if (objects != null)
            {
                foreach (var obj in objects)
                {
                    foreach (var property in properties)
                    {
                        writer.WriteBinaryValue(property.PropertyType, property.Getter()(obj));
                    }
                    writer.WriteOriginalValues(obj);
                }
            }
        }

        /// <summary>
        /// The inverse of <see cref="WriteObjects{TObject}(CoreBinaryWriter, ICollection{TObject})"/>.
        /// </summary>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="reader"></param>
        /// <returns></returns>
        public static List<TObject> ReadObjects<TObject>(this CoreBinaryReader reader)
            where TObject : BaseObject, new()
        {
            return ReadObjects<TObject>(reader, typeof(TObject));
        }

        public static List<TObject> ReadObjects<TObject>(this CoreBinaryReader reader, Type type) where TObject : BaseObject
        {
            if (!typeof(TObject).IsAssignableFrom(type))
                throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}");

            var objs = new List<TObject>();
            var properties = new List<IProperty>();

            var nObjs = reader.ReadInt32();
            if (nObjs == 0)
            {
                return objs;
            }

            var nProps = reader.ReadInt32();
            for (int i = 0; i < nProps; ++i)
            {
                var propertyName = reader.ReadString();
                var property = DatabaseSchema.Property(type, propertyName)
                    ?? throw new SerialisationException($"Property {propertyName} does not exist on {type.EntityName()}");
                properties.Add(property);
            }

            for (int i = 0; i < nObjs; ++i)
            {
                var obj = (Activator.CreateInstance(type) as TObject)!;
                obj.SetObserving(false);

                foreach (var property in properties)
                {
                    property.Setter()(obj, reader.ReadBinaryValue(property.PropertyType));
                }
                reader.ReadOriginalValues(obj);

                obj.SetObserving(true);

                objs.Add(obj);
            }
            return objs;
        }
    }

    /// <summary>
    /// When serialising an object implementing this interface, a '$type' field will be added.
    /// </summary>
    public interface IPolymorphicallySerialisable { }

    /// <summary>
    /// Adds a '$type' property to all classes that implement <see cref="IPolymorphicallySerialisable"/>.
    /// </summary>
    public class PolymorphicConverter : JsonConverter<object>
    {
        public override bool CanConvert(Type typeToConvert)
        {
            return typeof(IPolymorphicallySerialisable).IsAssignableFrom(typeToConvert) && (typeToConvert.IsInterface || typeToConvert.IsAbstract);
        }

        public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
        {
            var dictionary = JsonSerializer.Deserialize<Dictionary<string, JsonElement>>(ref reader, options);
            if (dictionary is null) return null;
            if(dictionary.TryGetValue("$type", out var typeName))
            {
                var type = Type.GetType(typeName.GetString() ?? "")!;
                dictionary.Remove("$type");
                var data = JsonSerializer.Serialize(dictionary, options);
                return JsonSerializer.Deserialize(data, type, options);
            }
            else
            {
                return null;
            }
        }

        public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options)
        {
            writer.WriteStartObject();
            writer.WriteString("$type", value.GetType().AssemblyQualifiedName);
            var internalSerialisation = JsonSerializer.Serialize(value, options)[1..^1];
            if (!internalSerialisation.IsNullOrWhiteSpace())
            {
                writer.WriteRawValue(internalSerialisation, true);
            }
            writer.WriteEndObject();
        }
    }

    public abstract class CustomJsonConverter<T> : JsonConverter<T>
    {
        protected object? ReadJson(ref Utf8JsonReader reader)
        {
            switch (reader.TokenType)
            {
                case JsonTokenType.String:
                    return reader.GetString();
                case JsonTokenType.Number:
                    if (reader.TryGetInt32(out int intValue))
                        return intValue;
                    if (reader.TryGetDouble(out double doubleValue))
                        return doubleValue;
                    return null;
                case JsonTokenType.True:
                    return true;
                case JsonTokenType.False:
                    return false;
                case JsonTokenType.Null:
                    return null;
                case JsonTokenType.StartArray:
                    var values = new List<object?>();
                    reader.Read();
                    while(reader.TokenType != JsonTokenType.EndArray)
                    {
                        values.Add(ReadJson(ref reader));
                        reader.Read();
                    }
                    return values;
                default:
                    return null;
            }
        }

        protected T ReadEnum<T>(ref Utf8JsonReader reader)
            where T : struct
        {
            if(reader.TokenType == JsonTokenType.Number)
            {
                return (T)Enum.ToObject(typeof(T), reader.GetInt32());
            }
            else
            {
                return Enum.Parse<T>(reader.GetString());
            }
        }

        protected delegate void ArrayValueHandler(ref Utf8JsonReader reader);

        protected void ReadArray(ref Utf8JsonReader reader, ArrayValueHandler onValue)
        {
            if (reader.TokenType != JsonTokenType.StartArray)
            {
                throw new JsonException();
            }
            while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
            {
                onValue(ref reader);
            }
        }

        protected delegate void ObjectPropertyHandler(ref Utf8JsonReader reader, string propertyName);

        protected void ReadObject(ref Utf8JsonReader reader, ObjectPropertyHandler onProperty)
        {
            if (reader.TokenType != JsonTokenType.StartObject)
            {
                throw new JsonException();
            }
            while (reader.Read() && reader.TokenType != JsonTokenType.EndObject)
            {
                if(reader.TokenType != JsonTokenType.PropertyName)
                {
                    throw new JsonException();
                }
                var property = reader.GetString() ?? "";
                reader.Read();
                onProperty(ref reader, property);
            }
        }

        
        /// <summary>
        /// Write a value as a JSON object; note that some data types, like
        /// <see cref="Guid"/> and <see cref="DateTime"/> will be encoded as
        /// strings, and therefore will be returned as strings when read by
        /// <see cref="ReadJson(Utf8JsonReader)"/>. However, all types that
        /// this can write should be able to be retrieved by calling <see
        /// cref="CoreUtils.ChangeType(object?, Type)"/> on the resultant
        /// value.
        /// </summary>
        protected void WriteJson(Utf8JsonWriter writer, object? value)
        {
            if  (value == null)
                writer.WriteNullValue();
            else if (value is string sVal)
                writer.WriteStringValue(sVal);
            else if (value is bool bVal)
                writer.WriteBooleanValue(bVal);
            else if (value is byte b)
                writer.WriteNumberValue(b);
            else if (value is short i16)
                writer.WriteNumberValue(i16);
            else if (value is int i32)
                writer.WriteNumberValue(i32);
            else if (value is long i64)
                writer.WriteNumberValue(i64);
            else if (value is float f)
                writer.WriteNumberValue(f);
            else if (value is double dVal)
                writer.WriteNumberValue(dVal);
            else if (value is DateTime dtVal)
                writer.WriteStringValue(dtVal.ToString());
            else if (value is TimeSpan tsVal)
                writer.WriteStringValue(tsVal.ToString());
            else if (value is Guid guid)
                writer.WriteStringValue(guid.ToString());
            else if(value is byte[] arr)
            {
                writer.WriteBase64StringValue(arr);
            }
            else if(value is Array array)
            {
                writer.WriteStartArray();
                foreach(var val1 in array)
                {
                    WriteJson(writer, val1);
                }
                writer.WriteEndArray();
            }
            else if(value is Enum e)
            {
                WriteJson(writer, Convert.ChangeType(e, e.GetType().GetEnumUnderlyingType()));
            }
            else
            {
                Logger.Send(LogType.Error, "", $"Could not write object of type {value.GetType()} as JSON");
            }
        }
        
        protected void WriteJson(Utf8JsonWriter writer, string name, object? value)
        {
            writer.WritePropertyName(name);
            WriteJson(writer, value);
        }
    }

    public class ObjectConverter : CustomJsonConverter<object?>
    {
        public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
        {
            switch (reader.TokenType)
            {
                case JsonTokenType.StartObject:
                    var dict = new Dictionary<string, object?>();
                    ReadObject(ref reader, (ref Utf8JsonReader reader, string property) =>
                    {
                        dict[property] = Read(ref reader, typeof(object), options);
                    });
                    return dict;
                case JsonTokenType.StartArray:
                    var list = new List<object?>();
                    ReadArray(ref reader, (ref Utf8JsonReader reader) =>
                    {
                        list.Add(Read(ref reader, typeof(object), options));
                    });
                    return list.ToArray();
                case JsonTokenType.String:
                    return reader.GetString();
                case JsonTokenType.False:
                    return false;
                case JsonTokenType.True:
                    return true;
                case JsonTokenType.Number:
                    if(reader.TryGetInt32(out var iValue))
                    {
                        return iValue;
                    }
                    else if(reader.TryGetInt64(out var lValue))
                    {
                        return lValue;
                    }
                    else if(reader.TryGetDouble(out var dValue))
                    {
                        return dValue;
                    }
                    else
                    {
                        return null;
                    }
                case JsonTokenType.Null:
                    return null;
                default:
                    throw new JsonException();
            }
        }

        public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerOptions options)
        {
            if(value is null)
            {
                writer.WriteNullValue();
            }
            else if(value.GetType() == typeof(object))
            {
                writer.WriteStartObject();
                writer.WriteEndObject();
            }
            else
            {
                // Call the serialiser, but this time with the value's real type, so this particular won't get called (since this only
                // gets called if the type passed into 'Serialize' is *identically* 'object'.
                JsonSerializer.Serialize(writer, value, value.GetType(), options);
            }
        }
    }

    public class TypeJsonConverter : CustomJsonConverter<Type>
    {
        public override Type? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
        {
            if(reader.TokenType == JsonTokenType.String)
            {
                return Type.GetType(reader.GetString());
            }
            else
            {
                return null;
            }
        }

        public override void Write(Utf8JsonWriter writer, Type value, JsonSerializerOptions options)
        {
            writer.WriteStringValue(value.AssemblyQualifiedName);
        }
    }
}