inabox/InABox.Core/Serialization.cs

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Net.WebSockets;
using System.Reflection;
using System.Runtime.InteropServices.ComTypes;
using System.Threading;
using System.Xml.Linq;
using InABox.Clients;
using JetBrains.Annotations;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace InABox.Core
{
    public interface ISerializeBinary
    {
        public void SerializeBinary(CoreBinaryWriter writer);

        public void DeserializeBinary(CoreBinaryReader reader);
    }

    public static class Serialization
    {
        private static JsonSerializerSettings? _serializerSettings;

        private static JsonSerializerSettings SerializerSettings(bool indented = true)
        {
            if (_serializerSettings == null)
            {
                _serializerSettings = new JsonSerializerSettings
                {
                    DateParseHandling = DateParseHandling.DateTime,
                    DateFormatHandling = DateFormatHandling.IsoDateFormat,
                    DateTimeZoneHandling = DateTimeZoneHandling.RoundtripKind
                };

                _serializerSettings.Converters.Add(new CoreTableJsonConverter());
                //serializerSettings.Converters.Add(new DateTimeJsonConverter());

                _serializerSettings.Converters.Add(new FilterJsonConverter());
                _serializerSettings.Converters.Add(new ColumnJsonConverter());
                _serializerSettings.Converters.Add(new SortOrderJsonConverter());
                _serializerSettings.Converters.Add(new UserPropertiesJsonConverter());
            }
            _serializerSettings.Formatting = indented ? Formatting.Indented : Formatting.None;

            return _serializerSettings;
        }

        public static string Serialize(object? o, bool indented = false)
        {
            var json = JsonConvert.SerializeObject(o, SerializerSettings(indented));
            return json;
        }

        public static void Serialize(object o, Stream stream, bool indented = false)
        {
            var settings = SerializerSettings(indented);
            using (var sw = new StreamWriter(stream))
            {
                using (JsonWriter writer = new JsonTextWriter(sw))
                {
                    var serializer = JsonSerializer.Create(settings);
                    serializer.Serialize(writer, o);
                }
            }
        }

        public static void DeserializeInto(string json, object target)
        {
            JsonConvert.PopulateObject(json, target, SerializerSettings());
        }

        [return: MaybeNull]
        public static T Deserialize<T>(Stream? stream, bool strict = false)
        {
            if (stream == null)
                return default;

            try
            {
                var settings = SerializerSettings();
                using var sr = new StreamReader(stream);
                using JsonReader reader = new JsonTextReader(sr);
                var serializer = JsonSerializer.Create(settings);
                return serializer.Deserialize<T>(reader);
            }
            catch (Exception e)
            {
                if (strict)
                    throw;

                Logger.Send(LogType.Error, ClientFactory.UserID, $"Error in Deserialize<{typeof(T)}>(): {e.Message}");
                return default;
            }
        }

        public static object? Deserialize(Type type, Stream? stream)
        {
            if (stream == null)
                return null;

            object? result = null;
            var settings = SerializerSettings();
            using (var sr = new StreamReader(stream))
            {
                using (JsonReader reader = new JsonTextReader(sr))
                {
                    var serializer = JsonSerializer.Create(settings);
                    result = serializer.Deserialize(reader, type);
                }
            }

            return result;
        }

        [return: MaybeNull]
        public static T Deserialize<T>(string? json, bool strict = false) // where T : new()
        {
            var ret = default(T);
            if (string.IsNullOrWhiteSpace(json))
                return ret;

            try
            {
                var settings = SerializerSettings();
                //if (typeof(T).IsSubclassOf(typeof(BaseObject)))
                //{
                //    ret = Activator.CreateInstance<T>();
                //    (ret as BaseObject).SetObserving(false);
                //    JsonConvert.PopulateObject(json, ret, settings);
                //    (ret as BaseObject).SetObserving(true);
                //}
                //else 
                if (typeof(T).IsArray)
                {
                    ret = JsonConvert.DeserializeObject<T>(json, settings);
                    //object o = Array.CreateInstance(typeof(T).GetElementType(), 0);
                    //ret = (T)o;
                }
                else
                {
                    ret = JsonConvert.DeserializeObject<T>(json, settings);
                }
            }
            catch (Exception)
            {
                if (strict)
                {
                    throw;
                }
                ret = Activator.CreateInstance<T>();
            }

            return ret;
        }

        public static object? Deserialize(Type T, string json) // where T : new()
        {
            var ret = T.GetDefault();
            if (string.IsNullOrWhiteSpace(json))
                return ret;

            try
            {
                var settings = SerializerSettings();
                //if (typeof(T).IsSubclassOf(typeof(BaseObject)))
                //{
                //    ret = Activator.CreateInstance<T>();
                //    (ret as BaseObject).SetObserving(false);
                //    JsonConvert.PopulateObject(json, ret, settings);
                //    (ret as BaseObject).SetObserving(true);
                //}
                //else 
                if (T.IsArray)
                {
                    object o = Array.CreateInstance(T.GetElementType(), 0);
                    ret = o;
                }
                else
                {
                    ret = JsonConvert.DeserializeObject(json, T, settings);
                }
            }
            catch (Exception)
            {
                ret = Activator.CreateInstance(T);
            }

            return ret;
        }

        #region Binary Serialization

        public static byte[] WriteBinary(this ISerializeBinary obj, BinarySerializationSettings settings)
        {
            using var stream = new MemoryStream();
            obj.SerializeBinary(new CoreBinaryWriter(stream, settings));

            return stream.ToArray();
        }

        public static T ReadBinary<T>(byte[] data, BinarySerializationSettings settings)
            where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), data, settings);
        public static T ReadBinary<T>(Stream stream, BinarySerializationSettings settings)
            where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), stream, settings);

        public static object ReadBinary(Type T, byte[] data, BinarySerializationSettings settings)
        {
            using var stream = new MemoryStream(data);
            return ReadBinary(T, stream, settings);
        }
        public static object ReadBinary(Type T, Stream stream, BinarySerializationSettings settings)
        {
            var obj = (Activator.CreateInstance(T) as ISerializeBinary)!;
            obj.DeserializeBinary(new CoreBinaryReader(stream, settings));
            return obj;
        }

        #endregion
    }

    public class CoreBinaryReader : BinaryReader
    {
        public BinarySerializationSettings Settings { get; set; }

        public CoreBinaryReader(Stream stream, BinarySerializationSettings settings) : base(stream)
        {
            Settings = settings;
        }
    }
    public class CoreBinaryWriter : BinaryWriter
    {
        public BinarySerializationSettings Settings { get; set; }

        public CoreBinaryWriter(Stream stream, BinarySerializationSettings settings) : base(stream)
        {
            Settings = settings;
        }
    }

    /// <summary>
    /// A class to maintain the consistency of serialisation formats across versions.
    /// The design of this is such that specific versions of serialisation have different parameters set,
    /// and the versions are maintained as static properties. Please keep the constructor private.
    /// </summary>
    /// <remarks>
    /// Note that <see cref="Latest"/> should always be updated to point to the latest version.
    /// <br/>
    /// Note also that all versions should have an entry in the <see cref="ConvertVersionString(string)"/> function.
    /// <br/>
    /// Also, if you create a new format, it would probably be a good idea to add a database update script to get all
    /// <see cref="IPackable"/> and <see cref="ISerializeBinary"/> properties and update the version of the format.
    /// (Otherwise, we'd basically be nullifying all data that is currently binary serialised.)
    /// </remarks>
    public class BinarySerializationSettings
    {
        /// <summary>
        /// Should reference types include a flag for nullability? (Adds an extra boolean field for whether the value is null or not).
        /// </summary>
        /// <remarks>
        /// True in all serialisation versions >= 1.1.
        /// </remarks>
        public bool IncludeNullables { get; set; }
        public string Version { get; set; }

        /// <summary>
        /// Handle correctly arrays of <see cref="object"/> inside of <see cref="Filter{T}.Value"/>.
        /// </summary>
        /// <remarks>
        /// True in all serialisation versions >= 1.2.
        /// </remarks>
        public bool HandleFilterObjectArrays { get; set; }

        public static BinarySerializationSettings Latest => V1_2;

        public static BinarySerializationSettings V1_0 = new BinarySerializationSettings("1.0")
        {
            IncludeNullables = false,
            HandleFilterObjectArrays = false
        };
        public static BinarySerializationSettings V1_1 = new BinarySerializationSettings("1.1")
        {
            IncludeNullables = true,
            HandleFilterObjectArrays = false
        };
        public static BinarySerializationSettings V1_2 = new BinarySerializationSettings("1.2")
        {
            IncludeNullables = true,
            HandleFilterObjectArrays = true
        };

        public static BinarySerializationSettings ConvertVersionString(string version) => version switch
        {
            "1.0" => V1_0,
            "1.1" => V1_1,
            "1.2" => V1_2,
            _ => V1_0
        };

        private BinarySerializationSettings(string version)
        {
            Version = version;
        }
    }

    public static class SerializationUtils
    {
        public static void Write(this BinaryWriter writer, Guid guid)
        {
            writer.Write(guid.ToByteArray());
        }
        public static Guid ReadGuid(this BinaryReader reader)
        {
            return new Guid(reader.ReadBytes(16));
        }

        /// <summary>
        /// Binary serialize a bunch of different types of values. <see cref="WriteBinaryValue(CoreBinaryWriter, Type, object?)"/> and
        /// <see cref="ReadBinaryValue(CoreBinaryReader, Type)"/> are inverses of each other.
        /// </summary>
        /// <remarks>
        /// Handles <see cref="byte"/>[], <see cref="Array"/>s of serialisable values, <see cref="Enum"/>, <see cref="bool"/>, <see cref="string"/>,
        /// <see cref="Guid"/>, <see cref="byte"/>, <see cref="Int16"/>, <see cref="Int32"/>, <see cref="Int64"/>, <see cref="float"/>, <see cref="double"/>,
        /// <see cref="DateTime"/>, <see cref="TimeSpan"/>, <see cref="LoggablePropertyAttribute"/>, <see cref="IPackable"/>, <see cref="Nullable{T}"/>
        /// and <see cref="ISerializeBinary"/>.
        /// </remarks>
        /// <param name="writer"></param>
        /// <param name="type"></param>
        /// <param name="value"></param>
        /// <exception cref="Exception">If an object of <paramref name="type"/> is unable to be serialized.</exception>
        public static void WriteBinaryValue(this CoreBinaryWriter writer, Type type, object? value)
        {
            value ??= CoreUtils.GetDefault(type);
            if (type == typeof(byte[]) && value is byte[] bArray)
            {
                writer.Write(bArray.Length);
                writer.Write(bArray);
            }
            else if (type == typeof(byte[]) && value is null)
            {
                writer.Write(0);
            }
            else if (type.IsArray && value is Array array)
            {
                var elementType = type.GetElementType();
                writer.Write(array.Length);
                foreach (var val1 in array)
                {
                    WriteBinaryValue(writer, elementType, val1);
                }
            }
            else if (type.IsArray && value is null)
            {
                writer.Write(0);
            }
            else if (type.IsEnum && value is Enum e)
            {
                var underlyingType = type.GetEnumUnderlyingType();
                WriteBinaryValue(writer, underlyingType, Convert.ChangeType(e, underlyingType));
            }
            else if (type == typeof(bool) && value is bool b)
            {
                writer.Write(b);
            }
            else if (type == typeof(string) && value is string str)
            {
                writer.Write(str);
            }
            else if (type == typeof(string) && value is null)
            {
                writer.Write("");
            }
            else if (type == typeof(Guid) && value is Guid guid)
            {
                writer.Write(guid);
            }
            else if (type == typeof(byte) && value is byte i8)
            {
                writer.Write(i8);
            }
            else if (type == typeof(Int16) && value is Int16 i16)
            {
                writer.Write(i16);
            }
            else if (type == typeof(Int32) && value is Int32 i32)
            {
                writer.Write(i32);
            }
            else if (type == typeof(Int64) && value is Int64 i64)
            {
                writer.Write(i64);
            }
            else if (type == typeof(float) && value is float f32)
            {
                writer.Write(f32);
            }
            else if (type == typeof(double) && value is double f64)
            {
                writer.Write(f64);
            }
            else if (type == typeof(DateTime) && value is DateTime date)
            {
                writer.Write(date.Ticks);
            }
            else if (type == typeof(TimeSpan) && value is TimeSpan time)
            {
                writer.Write(time.Ticks);
            }
            else if (type == typeof(LoggablePropertyAttribute))
            {
                writer.Write((value as LoggablePropertyAttribute)?.Format ?? "");
            }
            else if (typeof(IPackable).IsAssignableFrom(type) && value is IPackable pack)
            {
                if (writer.Settings.IncludeNullables)
                {
                    writer.Write(true);
                }
                pack.Pack(writer);
            }
            else if (writer.Settings.IncludeNullables && typeof(IPackable).IsAssignableFrom(type) && value is null)
            {
                writer.Write(false);
            }
            else if (typeof(ISerializeBinary).IsAssignableFrom(type) && value is ISerializeBinary binary)
            {
                if (writer.Settings.IncludeNullables)
                {
                    writer.Write(true);
                }
                binary.SerializeBinary(writer);
            }
            else if (writer.Settings.IncludeNullables && typeof(ISerializeBinary).IsAssignableFrom(type) && value is null)
            {
                writer.Write(false);
            }
            else if (Nullable.GetUnderlyingType(type) is Type t)
            {
                if (value == null)
                {
                    writer.Write(false);
                }
                else
                {
                    writer.Write(true);
                    writer.WriteBinaryValue(t, value);
                }
            }
            else if (value is UserProperty userprop)
            {
                 WriteBinaryValue(writer, userprop.Type, userprop.Value);
            }
            else
            {
                throw new Exception($"Invalid type; Target DataType is {type} and value DataType is {value?.GetType().ToString() ?? "null"}");
            }
        }

        /// <summary>
        /// Binary deserialize a bunch of different types of values. <see cref="WriteBinaryValue(CoreBinaryWriter, Type, object?)"/> and
        /// <see cref="ReadBinaryValue(CoreBinaryReader, Type)"/> are inverses of each other.
        /// </summary>
        /// <remarks>
        /// Handles <see cref="byte"/>[], <see cref="Array"/>s of serialisable values, <see cref="Enum"/>, <see cref="bool"/>, <see cref="string"/>,
        /// <see cref="Guid"/>, <see cref="byte"/>, <see cref="Int16"/>, <see cref="Int32"/>, <see cref="Int64"/>, <see cref="float"/>, <see cref="double"/>,
        /// <see cref="DateTime"/>, <see cref="TimeSpan"/>, <see cref="LoggablePropertyAttribute"/>, <see cref="IPackable"/>, <see cref="Nullable{T}"/>
        /// and <see cref="ISerializeBinary"/>.
        /// </remarks>
        /// <param name="reader"></param>
        /// <param name="type"></param>
        /// <exception cref="Exception">If an object of <paramref name="type"/> is unable to be deserialized.</exception>
        public static object? ReadBinaryValue(this CoreBinaryReader reader, Type type)
        {
            if (type == typeof(byte[]))
            {
                var length = reader.ReadInt32();
                return reader.ReadBytes(length);
            }
            else if (type.IsArray)
            {
                var length = reader.ReadInt32();
                var elementType = type.GetElementType();

                var array = Array.CreateInstance(elementType, length);
                for (int i = 0; i < array.Length; ++i)
                {
                    array.SetValue(ReadBinaryValue(reader, elementType), i);
                }
                return array;
            }
            else if (type.IsEnum)
            {
                var val = ReadBinaryValue(reader, type.GetEnumUnderlyingType());
                return Enum.ToObject(type, val);
            }
            else if (type == typeof(bool))
            {
                return reader.ReadBoolean();
            }
            else if (type == typeof(string))
            {
                return reader.ReadString();
            }
            else if (type == typeof(Guid))
            {
                return reader.ReadGuid();
            }
            else if (type == typeof(byte))
            {
                return reader.ReadByte();
            }
            else if (type == typeof(Int16))
            {
                return reader.ReadInt16();
            }
            else if (type == typeof(Int32))
            {
                return reader.ReadInt32();
            }
            else if (type == typeof(Int64))
            {
                return reader.ReadInt64();
            }
            else if (type == typeof(float))
            {
                return reader.ReadSingle();
            }
            else if (type == typeof(double))
            {
                return reader.ReadDouble();
            }
            else if (type == typeof(DateTime))
            {
                return new DateTime(reader.ReadInt64());
            }
            else if (type == typeof(TimeSpan))
            {
                return new TimeSpan(reader.ReadInt64());
            }
            else if (type == typeof(LoggablePropertyAttribute))
            {
                String format = reader.ReadString();
                return String.IsNullOrWhiteSpace(format)
                    ? null
                    : new LoggablePropertyAttribute() { Format = format };
            }
            else if (typeof(IPackable).IsAssignableFrom(type))
            {
                if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean.
                {
                    var packable = (Activator.CreateInstance(type) as IPackable)!;
                    packable.Unpack(reader);
                    return packable;
                }
                else
                {
                    return null;
                }
            }
            else if (typeof(ISerializeBinary).IsAssignableFrom(type))
            {
                if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean.
                {
                    var obj = (Activator.CreateInstance(type) as ISerializeBinary)!;
                    obj.DeserializeBinary(reader);
                    return obj;
                }
                else
                {
                    return null;
                }
            }
            else if (Nullable.GetUnderlyingType(type) is Type t)
            {
                var isNull = reader.ReadBoolean();
                if (isNull)
                {
                    return null;
                }
                else
                {
                    return reader.ReadBinaryValue(t);
                }
            }
            else
            {
                throw new Exception($"Invalid type; Target DataType is {type}");
            }
        }

        public static IEnumerable<IProperty> SerializableProperties(Type type) =>
            DatabaseSchema.Properties(type)
                .Where(x => !(x is StandardProperty st) || st.Property.GetCustomAttribute<DoNotSerialize>() == null);

        private static void GetOriginalValues(BaseObject obj, string? parent, List<Tuple<Type, string, object?>> values)
        {
            parent = parent != null ? $"{parent}." : "";

            foreach (var (key, value) in obj.OriginalValues)
            {
                if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop)
                {
                    values.Add(new Tuple<Type, string, object?>(prop.PropertyType, parent + key, value));
                }
            }
            var props = obj.GetType().GetProperties().Where(x =>
                x.GetCustomAttribute<DoNotSerialize>() == null
                && x.GetCustomAttribute<DoNotPersist>() == null
                && x.GetCustomAttribute<AggregateAttribute>() == null
                && x.GetCustomAttribute<FormulaAttribute>() == null
                && x.GetCustomAttribute<ConditionAttribute>() == null
                && x.CanWrite);
            foreach (var prop in props)
            {
                if (prop.PropertyType.GetInterfaces().Contains(typeof(IEnclosedEntity)))
                {
                    if (prop.GetValue(obj) is BaseObject child)
                        GetOriginalValues(child, parent + prop.Name, values);
                }
                else if (prop.PropertyType.GetInterfaces().Contains(typeof(IEntityLink)))
                {
                    if (prop.GetValue(obj) is BaseObject child && child.HasOriginalValue("ID"))
                    {
                        values.Add(new Tuple<Type, string, object?>(typeof(Guid), parent + prop.Name + ".ID", child.OriginalValues["ID"]));
                    }
                }
            }
        }

        private static void WriteOriginalValues<TObject>(this CoreBinaryWriter writer, TObject obj)
            where TObject : BaseObject
        {
            var originalValues = new List<Tuple<Type, string, object?>>();
            GetOriginalValues(obj, null, originalValues);

            writer.Write(originalValues.Count);
            foreach (var (type, key, value) in originalValues)
            {
                writer.Write(key);
                writer.WriteBinaryValue(type, value);
            }
        }
        private static void ReadOriginalValues<TObject>(this CoreBinaryReader reader, TObject obj)
            where TObject : BaseObject
        {
            var nOriginalValues = reader.ReadInt32();
            for (int i = 0; i < nOriginalValues; ++i)
            {
                var key = reader.ReadString();
                if (DatabaseSchema.Property(typeof(TObject), key) is IProperty prop)
                {
                    var value = reader.ReadBinaryValue(prop.PropertyType);
                    if (prop.Parent is null)
                    {
                        obj.OriginalValues[prop.Name] = value;
                    }
                    else
                    {
                        if (prop.Parent.Getter()(obj) is BaseObject parent)
                        {
                            parent.OriginalValues[prop.Name.Split('.').Last()] = value;
                        }
                    }
                }
            }
        }

        /// <summary>
        /// An implementation of binary serialising a <typeparamref name="TObject"/>; this is the inverse of <see cref="ReadObject{TObject}(CoreBinaryReader)"/>.
        /// </summary>
        /// <remarks>
        /// Also serialises the names of properties along with the values.
        /// </remarks>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="writer"></param>
        /// <param name="entity"></param>
        public static void WriteObject<TObject>(this CoreBinaryWriter writer, TObject entity)
            where TObject : BaseObject, new()
        {
            var properties = SerializableProperties(typeof(TObject)).ToList();

            writer.Write(properties.Count);
            foreach (var property in properties)
            {
                writer.Write(property.Name);
                writer.WriteBinaryValue(property.PropertyType, property.Getter()(entity));
            }
            writer.WriteOriginalValues(entity);
        }

        /// <summary>
        /// The inverse of <see cref="WriteObject{TObject}(CoreBinaryWriter, TObject)"/>.
        /// </summary>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="reader"></param>
        /// <returns></returns>
        public static TObject ReadObject<TObject>(this CoreBinaryReader reader)
            where TObject : BaseObject, new()
        {
            var obj = new TObject();
            obj.SetObserving(false);

            var nProps = reader.ReadInt32();
            for (int i = 0; i < nProps; ++i)
            {
                var propName = reader.ReadString();
                var property = DatabaseSchema.Property(typeof(TObject), propName);
                property?.Setter()(obj, reader.ReadBinaryValue(property.PropertyType));
            }
            reader.ReadOriginalValues(obj);

            obj.SetObserving(true);

            return obj;
        }

        /// <summary>
        /// An implementation of binary serialising multiple <typeparamref name="TObject"/>s;
        /// this is the inverse of <see cref="ReadObjects{TObject}(CoreBinaryReader)"/>.
        /// </summary>
        /// <remarks>
        /// Also serialises the names of properties along with the values.
        /// </remarks>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="writer"></param>
        /// <param name="objects"></param>
        public static void WriteObjects<TObject>(this CoreBinaryWriter writer, ICollection<TObject> objects)
            where TObject : BaseObject, new()
        {
            var properties = SerializableProperties(typeof(TObject)).ToList();

            writer.Write(objects.Count);
            writer.Write(properties.Count);
            foreach (var property in properties)
            {
                writer.Write(property.Name);
            }

            foreach (var obj in objects)
            {
                foreach (var property in properties)
                {
                    writer.WriteBinaryValue(property.PropertyType, property.Getter()(obj));
                }
                writer.WriteOriginalValues(obj);
            }
        }

        /// <summary>
        /// The inverse of <see cref="WriteObjects{TObject}(CoreBinaryWriter, ICollection{TObject})"/>.
        /// </summary>
        /// <typeparam name="TObject"></typeparam>
        /// <param name="reader"></param>
        /// <returns></returns>
        public static List<TObject> ReadObjects<TObject>(this CoreBinaryReader reader)
            where TObject : BaseObject, new()
        {
            var objs = new List<TObject>();
            var properties = new List<IProperty>();

            var nObjs = reader.ReadInt32();
            var nProps = reader.ReadInt32();
            for (int i = 0; i < nProps; ++i)
            {
                var property = reader.ReadString();
                properties.Add(DatabaseSchema.Property(typeof(TObject), property));
            }

            for (int i = 0; i < nObjs; ++i)
            {
                var obj = new TObject();
                obj.SetObserving(false);

                foreach (var property in properties)
                {
                    property?.Setter()(obj, reader.ReadBinaryValue(property.PropertyType));
                }
                reader.ReadOriginalValues(obj);

                obj.SetObserving(true);

                objs.Add(obj);
            }
            return objs;
        }
    }
}