using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Net.WebSockets; using System.Reflection; using System.Runtime.InteropServices.ComTypes; using System.Threading; using System.Xml.Linq; using InABox.Clients; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace InABox.Core { public class SerialisationException : Exception { public SerialisationException(string message): base(message) { } } public interface ISerializeBinary { public void SerializeBinary(CoreBinaryWriter writer); public void DeserializeBinary(CoreBinaryReader reader); } public static class Serialization { private static JsonSerializerSettings? _serializerSettings; private static JsonSerializerSettings SerializerSettings(bool indented = true) { if (_serializerSettings == null) { _serializerSettings = new JsonSerializerSettings { DateParseHandling = DateParseHandling.DateTime, DateFormatHandling = DateFormatHandling.IsoDateFormat, DateTimeZoneHandling = DateTimeZoneHandling.RoundtripKind }; _serializerSettings.Converters.Add(new CoreTableJsonConverter()); //serializerSettings.Converters.Add(new DateTimeJsonConverter()); _serializerSettings.Converters.Add(new FilterJsonConverter()); _serializerSettings.Converters.Add(new ColumnJsonConverter()); _serializerSettings.Converters.Add(new SortOrderJsonConverter()); _serializerSettings.Converters.Add(new UserPropertiesJsonConverter()); _serializerSettings.ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor; } _serializerSettings.Formatting = indented ? Formatting.Indented : Formatting.None; return _serializerSettings; } public static string Serialize(object? o, bool indented = false) { var json = JsonConvert.SerializeObject(o, SerializerSettings(indented)); return json; } public static void Serialize(object o, Stream stream, bool indented = false) { var settings = SerializerSettings(indented); using (var sw = new StreamWriter(stream)) { using (JsonWriter writer = new JsonTextWriter(sw)) { var serializer = JsonSerializer.Create(settings); serializer.Serialize(writer, o); } } } public static void DeserializeInto(string json, object target) { JsonConvert.PopulateObject(json, target, SerializerSettings()); } [return: MaybeNull] public static T Deserialize(Stream? stream, bool strict = false) { if (stream == null) return default; try { var settings = SerializerSettings(); using var sr = new StreamReader(stream); using JsonReader reader = new JsonTextReader(sr); var serializer = JsonSerializer.Create(settings); return serializer.Deserialize(reader); } catch (Exception e) { if (strict) throw; Logger.Send(LogType.Error, ClientFactory.UserID, $"Error in Deserialize<{typeof(T)}>(): {e.Message}"); return default; } } public static object? Deserialize(Type type, Stream? stream) { if (stream == null) return null; object? result = null; var settings = SerializerSettings(); using (var sr = new StreamReader(stream)) { using (JsonReader reader = new JsonTextReader(sr)) { var serializer = JsonSerializer.Create(settings); result = serializer.Deserialize(reader, type); } } return result; } [return: MaybeNull] public static T Deserialize(JToken obj, bool strict = false) { var ret = default(T); try { var settings = SerializerSettings(); var serializer = JsonSerializer.Create(settings); return obj.ToObject(); } catch (Exception) { if (strict) { throw; } if (typeof(T).IsArray) { ret = (T)(object)Array.CreateInstance(typeof(T).GetElementType(), 0); } else { ret = Activator.CreateInstance(); } } return ret; } [return: MaybeNull] public static T Deserialize(string? json, bool strict = false) // where T : new() { var ret = default(T); if (string.IsNullOrWhiteSpace(json)) return ret; try { var settings = SerializerSettings(); //if (typeof(T).IsSubclassOf(typeof(BaseObject))) //{ // ret = Activator.CreateInstance(); // (ret as BaseObject).SetObserving(false); // JsonConvert.PopulateObject(json, ret, settings); // (ret as BaseObject).SetObserving(true); //} //else if (typeof(T).IsArray) { ret = JsonConvert.DeserializeObject(json, settings); //object o = Array.CreateInstance(typeof(T).GetElementType(), 0); //ret = (T)o; } else { ret = JsonConvert.DeserializeObject(json, settings); } } catch (Exception e) { if (strict) { throw; } if (typeof(T).IsArray) { ret = (T)(object)Array.CreateInstance(typeof(T).GetElementType(), 0); } else { ret = (T)Activator.CreateInstance(typeof(T), true); } } return ret; } public static object? Deserialize(Type T, string json) // where T : new() { var ret = T.GetDefault(); if (string.IsNullOrWhiteSpace(json)) return ret; try { var settings = SerializerSettings(); //if (typeof(T).IsSubclassOf(typeof(BaseObject))) //{ // ret = Activator.CreateInstance(); // (ret as BaseObject).SetObserving(false); // JsonConvert.PopulateObject(json, ret, settings); // (ret as BaseObject).SetObserving(true); //} //else if (T.IsArray) { object o = Array.CreateInstance(T.GetElementType(), 0); ret = o; } else { ret = JsonConvert.DeserializeObject(json, T, settings); } } catch (Exception) { ret = Activator.CreateInstance(T, true); } return ret; } #region Binary Serialization public static byte[] WriteBinary(this ISerializeBinary obj, BinarySerializationSettings settings) { using var stream = new MemoryStream(); obj.SerializeBinary(new CoreBinaryWriter(stream, settings)); return stream.ToArray(); } public static void WriteBinary(this ISerializeBinary obj, Stream stream, BinarySerializationSettings settings) { obj.SerializeBinary(new CoreBinaryWriter(stream, settings)); } public static T ReadBinary(byte[] data, BinarySerializationSettings settings) where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), data, settings); public static T ReadBinary(Stream stream, BinarySerializationSettings settings) where T : ISerializeBinary, new() => (T)ReadBinary(typeof(T), stream, settings); public static object ReadBinary(Type T, byte[] data, BinarySerializationSettings settings) { using var stream = new MemoryStream(data); return ReadBinary(T, stream, settings); } public static object ReadBinary(Type T, Stream stream, BinarySerializationSettings settings) { var obj = (Activator.CreateInstance(T) as ISerializeBinary)!; obj.DeserializeBinary(new CoreBinaryReader(stream, settings)); return obj; } #endregion } public class CoreBinaryReader : BinaryReader { public BinarySerializationSettings Settings { get; set; } public CoreBinaryReader(Stream stream, BinarySerializationSettings settings) : base(stream) { Settings = settings; } } public class CoreBinaryWriter : BinaryWriter { public BinarySerializationSettings Settings { get; set; } public CoreBinaryWriter(Stream stream, BinarySerializationSettings settings) : base(stream) { Settings = settings; } } /// /// A class to maintain the consistency of serialisation formats across versions. /// The design of this is such that specific versions of serialisation have different parameters set, /// and the versions are maintained as static properties. Please keep the constructor private. /// /// /// Note that should always be updated to point to the latest version. ///
/// Note also that all versions should have an entry in the function. ///
/// Also, if you create a new format, it would probably be a good idea to add a database update script to get all /// and properties and update the version of the format. /// (Otherwise, we'd basically be nullifying all data that is currently binary serialised.) ///
public class BinarySerializationSettings { /// /// Should the Info() call return RPC and Rest Ports? This is /// To workaround a bug in RPCsockets that crash on large uploads /// /// /// True in all serialization versions >= 1.2 /// public bool RPCClientWorkaround { get; set; } /// /// Should reference types include a flag for nullability? (Adds an extra boolean field for whether the value is null or not). /// /// /// True in all serialisation versions >= 1.1. /// public bool IncludeNullables { get; set; } public string Version { get; set; } public static BinarySerializationSettings Latest => V1_2; public static BinarySerializationSettings V1_0 = new BinarySerializationSettings("1.0") { IncludeNullables = false, RPCClientWorkaround = false }; public static BinarySerializationSettings V1_1 = new BinarySerializationSettings("1.1") { IncludeNullables = true, RPCClientWorkaround = false }; public static BinarySerializationSettings V1_2 = new BinarySerializationSettings("1.2") { IncludeNullables = true, RPCClientWorkaround = true }; public static BinarySerializationSettings ConvertVersionString(string version) => version switch { "1.0" => V1_0, "1.1" => V1_1, "1.2" => V1_2, _ => V1_0 }; private BinarySerializationSettings(string version) { Version = version; } } public static class SerializationUtils { public static void Write(this BinaryWriter writer, Guid guid) { writer.Write(guid.ToByteArray()); } public static Guid ReadGuid(this BinaryReader reader) { return new Guid(reader.ReadBytes(16)); } public static void Write(this BinaryWriter writer, DateTime dateTime) { writer.Write(dateTime.Ticks); } public static DateTime ReadDateTime(this BinaryReader reader) { return new DateTime(reader.ReadInt64()); } private static bool MatchType(Type t) => typeof(T1) == t; private static bool MatchType(Type t) => (typeof(T1) == t) || (typeof(T2) == t); /// /// Binary serialize a bunch of different types of values. and /// are inverses of each other. /// /// /// Handles [], s of serialisable values, , , , /// , , , , , , , /// , , , , /// and . /// /// /// /// /// If an object of is unable to be serialized. public static void WriteBinaryValue(this CoreBinaryWriter writer, Type type, object? value) { value ??= CoreUtils.GetDefault(type); if (value == null) { if (MatchType(type)) writer.Write(""); else if (writer.Settings.IncludeNullables && typeof(IPackable).IsAssignableFrom(type)) writer.Write(false); else if (writer.Settings.IncludeNullables && typeof(ISerializeBinary).IsAssignableFrom(type)) writer.Write(false); else if (Nullable.GetUnderlyingType(type) is Type t) writer.Write(false); else if (MatchType(type)) writer.Write(""); else writer.Write(0); } else if (MatchType(type) && value is byte[] bArray) { writer.Write(bArray.Length); writer.Write(bArray); } else if (type.IsArray && value is Array array) { var elementType = type.GetElementType(); writer.Write(array.Length); foreach (var val1 in array) { WriteBinaryValue(writer, elementType, val1); } } else if (type.IsEnum && value is Enum e) { var underlyingType = type.GetEnumUnderlyingType(); WriteBinaryValue(writer, underlyingType, Convert.ChangeType(e, underlyingType)); } else if (MatchType(type) && value is bool b) { writer.Write(b); } else if (MatchType(type) && value is string str) writer.Write(str); else if (MatchType(type) && value is Guid guid) writer.Write(guid); else if (MatchType(type) && value is byte i8) writer.Write(i8); else if (MatchType(type) && value is Int16 i16) writer.Write(i16); else if (MatchType(type) && value is Int32 i32) writer.Write(i32); else if (MatchType(type) && value is Int64 i64) writer.Write(i64); else if (MatchType(type) && value is float f32) writer.Write(f32); else if (MatchType(type) && value is double f64) writer.Write(f64); else if (MatchType(type) && value is DateTime date) writer.Write(date.Ticks); else if (MatchType(type) && value is TimeSpan time) writer.Write(time.Ticks); else if (MatchType(type) && value is LoggablePropertyAttribute lpa) writer.Write(lpa.Format ?? string.Empty); else if (typeof(IPackable).IsAssignableFrom(type) && value is IPackable pack) { if (writer.Settings.IncludeNullables) writer.Write(true); pack.Pack(writer); } else if (typeof(ISerializeBinary).IsAssignableFrom(type) && value is ISerializeBinary binary) { if (writer.Settings.IncludeNullables) writer.Write(true); binary.SerializeBinary(writer); } else if (Nullable.GetUnderlyingType(type) is Type t) { writer.Write(true); writer.WriteBinaryValue(t, value); } else if (value is UserProperty userprop) WriteBinaryValue(writer, userprop.Type, userprop.Value); else throw new SerialisationException($"Invalid type; Target DataType is {type} and value DataType is {value?.GetType().ToString() ?? "null"}"); } public static void WriteBinaryValue(this CoreBinaryWriter writer, T value) => WriteBinaryValue(writer, typeof(T), value); /// /// Binary deserialize a bunch of different types of values. and /// are inverses of each other. /// /// /// Handles [], s of serialisable values, , , , /// , , , , , , , /// , , , , /// and . /// /// /// /// If an object of is unable to be deserialized. public static object? ReadBinaryValue(this CoreBinaryReader reader, Type type) { if (type == typeof(byte[])) { var length = reader.ReadInt32(); return reader.ReadBytes(length); } else if (type.IsArray) { var length = reader.ReadInt32(); var elementType = type.GetElementType(); var array = Array.CreateInstance(elementType, length); for (int i = 0; i < array.Length; ++i) { array.SetValue(ReadBinaryValue(reader, elementType), i); } return array; } else if (type.IsEnum) { var val = ReadBinaryValue(reader, type.GetEnumUnderlyingType()); return Enum.ToObject(type, val); } else if (type == typeof(bool)) { return reader.ReadBoolean(); } else if (type == typeof(string)) { return reader.ReadString(); } else if (type == typeof(Guid)) { return reader.ReadGuid(); } else if (type == typeof(byte)) { return reader.ReadByte(); } else if (type == typeof(Int16)) { return reader.ReadInt16(); } else if (type == typeof(Int32)) { return reader.ReadInt32(); } else if (type == typeof(Int64)) { return reader.ReadInt64(); } else if (type == typeof(float)) { return reader.ReadSingle(); } else if (type == typeof(double)) { return reader.ReadDouble(); } else if (type == typeof(DateTime)) { return new DateTime(reader.ReadInt64()); } else if (type == typeof(TimeSpan)) { return new TimeSpan(reader.ReadInt64()); } else if (type == typeof(LoggablePropertyAttribute)) { String format = reader.ReadString(); return String.IsNullOrWhiteSpace(format) ? null : new LoggablePropertyAttribute() { Format = format }; } else if (typeof(IPackable).IsAssignableFrom(type)) { if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean. { var packable = (Activator.CreateInstance(type) as IPackable)!; packable.Unpack(reader); return packable; } else { return null; } } else if (typeof(ISerializeBinary).IsAssignableFrom(type)) { if (!reader.Settings.IncludeNullables || reader.ReadBoolean()) // Note the short-circuit operator preventing reading a boolean. { var obj = (Activator.CreateInstance(type, true) as ISerializeBinary)!; obj.DeserializeBinary(reader); return obj; } else { return null; } } else if (Nullable.GetUnderlyingType(type) is Type t) { var isNull = reader.ReadBoolean(); if (isNull) { return null; } else { return reader.ReadBinaryValue(t); } } else { throw new SerialisationException($"Invalid type; Target DataType is {type}"); } } public static T ReadBinaryValue(this CoreBinaryReader reader) { var result = ReadBinaryValue(reader, typeof(T)); return (result != null ? (T)result : default)!; } private static bool IsSerializable(Type type, StandardProperty? prop) { if (prop == null) return true; if (prop.Property.GetCustomAttribute() != null) return false; return IsSerializable(type, prop.Parent as StandardProperty); } public static IEnumerable SerializableProperties(Type type) => DatabaseSchema.Properties(type) .Where(x => !(x is StandardProperty st) || IsSerializable(type,st)); private static void GetOriginalValues(BaseObject obj, string? parent, List> values) { parent = parent != null ? $"{parent}." : ""; foreach (var (key, value) in obj.OriginalValues) { // EnclosedEntities and EntityLinks will be updated through the recursive code below, // so we should not need to serialise the entire object again.. if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop && !prop.PropertyType.GetInterfaces().Contains(typeof(IEnclosedEntity)) && !prop.PropertyType.GetInterfaces().Contains(typeof(IEntityLink)) ) { values.Add(new Tuple(prop.PropertyType, parent + key, value)); } } var props = obj.GetType().GetProperties().Where(x => x.GetCustomAttribute() == null && x.GetCustomAttribute() == null && x.GetCustomAttribute() == null && x.GetCustomAttribute() == null && x.GetCustomAttribute() == null && x.GetCustomAttribute() == null && x.GetCustomAttribute() == null && x.CanWrite); foreach (var prop in props) { if (prop.PropertyType.GetInterfaces().Contains(typeof(IEnclosedEntity))) { if (prop.GetValue(obj) is BaseObject child) GetOriginalValues(child, parent + prop.Name, values); } else if (prop.PropertyType.GetInterfaces().Contains(typeof(IEntityLink))) { if (prop.GetValue(obj) is BaseObject child && child.HasOriginalValue("ID")) { values.Add(new Tuple(typeof(Guid), parent + prop.Name + ".ID", child.OriginalValues["ID"])); } } } } private static void WriteOriginalValues(this CoreBinaryWriter writer, TObject obj) where TObject : BaseObject { var originalValues = new List>(); GetOriginalValues(obj, null, originalValues); writer.Write(originalValues.Count); foreach (var (type, key, value) in originalValues) { writer.Write(key); try { writer.WriteBinaryValue(type, value); } catch (Exception e) { } } } private static void ReadOriginalValues(this CoreBinaryReader reader, TObject obj) where TObject : BaseObject { var nOriginalValues = reader.ReadInt32(); for (int i = 0; i < nOriginalValues; ++i) { var key = reader.ReadString(); if (DatabaseSchema.Property(obj.GetType(), key) is IProperty prop) { var value = reader.ReadBinaryValue(prop.PropertyType); if (prop.Parent is null) { obj.OriginalValues[prop.Name] = value; } else { if (prop.Parent.Getter()(obj) is BaseObject parent) { parent.OriginalValues[prop.Name.Split('.').Last()] = value; } } } } } public static void WriteObject(this CoreBinaryWriter writer, TObject entity, Type type) where TObject : BaseObject { if (!typeof(TObject).IsAssignableFrom(type)) throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}"); var properties = SerializableProperties(type).ToList(); writer.Write(properties.Count); foreach (var property in properties) { writer.Write(property.Name); writer.WriteBinaryValue(property.PropertyType, property.Getter()(entity)); } writer.WriteOriginalValues(entity); } /// /// An implementation of binary serialising a ; this is the inverse of . /// /// /// Also serialises the names of properties along with the values. /// /// /// /// public static void WriteObject(this CoreBinaryWriter writer, TObject entity) where TObject : BaseObject, new() => WriteObject(writer, entity, typeof(TObject)); public static TObject ReadObject(this CoreBinaryReader reader, Type type) where TObject : BaseObject { if (!typeof(TObject).IsAssignableFrom(type)) throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}"); var obj = (Activator.CreateInstance(type) as TObject)!; obj.SetObserving(false); var nProps = reader.ReadInt32(); for (int i = 0; i < nProps; ++i) { var propName = reader.ReadString(); var property = DatabaseSchema.Property(type, propName) ?? throw new SerialisationException($"Property {propName} does not exist on {type.EntityName()}"); property.Setter()(obj, reader.ReadBinaryValue(property.PropertyType)); } reader.ReadOriginalValues(obj); obj.SetObserving(true); return obj; } /// /// The inverse of . /// /// /// /// public static TObject ReadObject(this CoreBinaryReader reader) where TObject : BaseObject, new() => reader.ReadObject(typeof(TObject)); /// /// An implementation of binary serialising multiple s; /// this is the inverse of . /// /// /// Also serialises the names of properties along with the values. /// /// /// /// public static void WriteObjects(this CoreBinaryWriter writer, ICollection? objects) where TObject : BaseObject, new() => WriteObjects(writer, typeof(TObject), objects); public static void WriteObjects(this CoreBinaryWriter writer, Type type, ICollection? objects) where TObject : BaseObject { if (!typeof(TObject).IsAssignableFrom(type)) throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}"); var nObjs = objects?.Count ?? 0; writer.Write(nObjs); if (nObjs == 0) { return; } var properties = SerializableProperties(type).ToList(); writer.Write(properties.Count); foreach (var property in properties) { writer.Write(property.Name); } if(objects != null) { foreach (var obj in objects) { foreach (var property in properties) { writer.WriteBinaryValue(property.PropertyType, property.Getter()(obj)); } writer.WriteOriginalValues(obj); } } } /// /// The inverse of . /// /// /// /// public static List ReadObjects(this CoreBinaryReader reader) where TObject : BaseObject, new() { return ReadObjects(reader, typeof(TObject)); } public static List ReadObjects(this CoreBinaryReader reader, Type type) where TObject : BaseObject { if (!typeof(TObject).IsAssignableFrom(type)) throw new SerialisationException($"{type.EntityName()} is not a subclass of {typeof(TObject).EntityName()}"); var objs = new List(); var properties = new List(); var nObjs = reader.ReadInt32(); if(nObjs == 0) { return objs; } var nProps = reader.ReadInt32(); for (int i = 0; i < nProps; ++i) { var propertyName = reader.ReadString(); var property = DatabaseSchema.Property(type, propertyName) ?? throw new SerialisationException($"Property {propertyName} does not exist on {type.EntityName()}"); properties.Add(property); } for (int i = 0; i < nObjs; ++i) { var obj = (Activator.CreateInstance(type) as TObject)!; obj.SetObserving(false); foreach (var property in properties) { property.Setter()(obj, reader.ReadBinaryValue(property.PropertyType)); } reader.ReadOriginalValues(obj); obj.SetObserving(true); objs.Add(obj); } return objs; } } }