// Use this define to validate the binary serialization. //#define Diagnose using System; using System.Diagnostics; using System.IO; using System.ComponentModel; using System.Linq; using System.Reflection; using System.Drawing; using System.Text; using JD.Core; namespace JD.Serialization { public class BinarySerializer : Serializer, IDisposable { private BigEndianBinaryReader _reader; private BigEndianBinaryWriter _writer; private readonly DebugTrace m_trace = new DebugTrace(); public uint DataVersion { get; set; } public BinarySerializer() : this(Serialization.TypeMap.Instance) { } protected BinarySerializer(ITypeMap typeMap) : base(typeMap) { } #region Read object from data Main protected override object LoadObjectFromFileImplementation(string file, Type type) { _reader = new BigEndianBinaryReader(File.Open(file, FileMode.Open)); return GetObjectFromReader(type, null); } public object LoadObjectFromBuffer(byte[] buffer, Type type, PropertyInfo property = null) { _reader = new BigEndianBinaryReader(new MemoryStream(buffer, 0, buffer.Length)); return GetObjectFromReader(type, property); } public uint GetDataVersion(Stream stream) { long position = stream.Position; _reader = new BigEndianBinaryReader(stream); ReadEngineVersion(); stream.Seek(position, SeekOrigin.Begin); return DataVersion; } public void Deserialize(Stream stream, object toFill, PropertyInfo property = null) { _reader = new BigEndianBinaryReader(stream); ReadEngineVersion(); FillObjectFromBin(toFill); } public object GetObjectFromReader(Type type, PropertyInfo property) { if (_reader != null) { ReadEngineVersion(); if (IsCollection(type)) return GetObjectList(type, property); else return CreateObjectFromType(type, property); } return null; } private void ReadEngineVersion() { DataVersion = Read(); } private void WriteEngineVersion() { Write(DataVersion); } #endregion #region Read object from data Internal private IUAFCollection GetObjectList(Type type, PropertyInfo property) { IUAFCollection collection = (IUAFCollection)Activator.CreateInstance(type); FillCollection(collection, property); return collection; } private void FillCollection(IUAFCollection collection, PropertyInfo property) { using (m_trace.StartList(property)) { bool factoryType = property.IsFactoryType(collection.ElementType); CheckRead(EngineType.SizeT, property); uint nbElements = _reader.ReadUInt32(); for (uint i = 0; i < nbElements; i++) { object value = null; if (factoryType) { var typeId = (StringID)CreateBaseTypeObject(typeof(StringID), property); var readType = ObjectFactory.getTypeFrom(typeId); if (readType == null) { throw new Exception($"Could not find type in '{collection.ElementType}' factory corresponding to CRC '{typeId.getValue()}'"); } value = CreateObjectFromType(readType, property); collection.Add(value); } else { value = CreateObjectFromType(collection.ElementType, property); collection.Add(value); } m_trace.Property(collection.Count - 1, value); } } } private T Read() { return (T)CreateObjectFromType(typeof(T)); } private void Write(T value) { WriteBaseTypeObject(value, typeof(T)); } public object CreateObjectFromType(Type type, PropertyInfo targetProperty = null, bool forceNoBaseType = false) { object obj; if (BasicTypeList.IsBasicType(type) && !forceNoBaseType) { return CreateBaseTypeObject(type, targetProperty); } else { obj = Activator.CreateInstance(TypeMap.ToRuntimeType(type)); FillObjectFromBin(obj); } return obj; } private object CreateBaseTypeObject(Type type, PropertyInfo targetProperty, bool checkReadType = true) { if (checkReadType) { CheckRead(type, targetProperty); } if (type.IsEnum) return _reader.ReadInt32(); object obj = null; TypeConverter conv = TypeDescriptor.GetConverter(type); switch (type.Name) { case "Boolean": return _reader.ReadInt32() != 0; case "int": case "Int32": return _reader.ReadInt32(); case "uint": case "UInt32": uint val = _reader.ReadUInt32(); return val; case "float": case "Single": return _reader.ReadSingle(); case "Double": return _reader.ReadDouble(); case "ObjectPath": return CreateObjectFromType(type, targetProperty, forceNoBaseType: true); case "String": case "String8": { UInt32 dirLen = _reader.ReadUInt32(); obj = Encoding.UTF8.GetString(_reader.ReadBytes((int)dirLen)); } break; case "Path": { UInt32 baseLen = _reader.ReadUInt32(); string baseStr = new string(_reader.ReadChars((int)baseLen)); UInt32 dirLen = _reader.ReadUInt32(); string dirStr = new string(_reader.ReadChars((int)dirLen)); _reader.ReadUInt32(); // stringID _reader.ReadUInt32(); // flag obj = dirStr + baseStr; } break; case "ColorInteger": obj = _reader.ReadUInt32(); break; case "StringID": obj = _reader.ReadUInt32(); if (IncludeSpecialFields) { obj = Encoding.UTF8.GetString(_reader.ReadBytes((int)(uint)obj)); } else if ((uint)obj == StringID.FullStringTag) { uint stringLength = _reader.ReadUInt32(); obj = Encoding.UTF8.GetString(_reader.ReadBytes((int)stringLength)); } break; case "LocalisationId": obj = _reader.ReadInt32(); break; case "Color": obj = String.Format("0x{3:X2}{2:X2}{1:X2}{0:X2}", (uint)(ColorF32ToU8(_reader.ReadSingle())), (uint)(ColorF32ToU8(_reader.ReadSingle())), (uint)(ColorF32ToU8(_reader.ReadSingle())), (uint)(ColorF32ToU8(_reader.ReadSingle()))); break; case "Angle": case "Volume": obj = _reader.ReadSingle(); break; case "Vec2d": { float x = _reader.ReadSingle(); float y = _reader.ReadSingle(); return new Vec2d(x, y); } case "Vec3d": { float x = _reader.ReadSingle(); float y = _reader.ReadSingle(); float z = _reader.ReadSingle(); return new Vec3d(x, y, z); } case "Platform": obj = new Platform((PlatformId)_reader.ReadUInt32()); break; } if (obj == null) return null; if (conv != null && conv.CanConvertFrom(obj.GetType())) { return conv.ConvertFrom(obj); } else { return Convert.ChangeType(obj, type); } } private bool IsPropertyPresent() { return _reader.ReadUInt32() != 0; } private void FillObjectFromBin(object obj) { using (m_trace.StartObject(obj)) { foreach (PropertyInfo property in SerializableFields(obj.GetType())) { if (property.IsOptional() && !IsPropertyPresent()) { property.SetValue(obj, null, null); continue; } if (BasicTypeList.IsBasicType(property.PropertyType)) { var propertyValue = CreateBaseTypeObject(property.PropertyType, property); property.SetValue(obj, propertyValue, null); m_trace.Property(property, propertyValue); continue; } if (IsCollection(property.PropertyType)) { IUAFCollection collection = (IUAFCollection)property.GetValue(obj, null); collection.Clear(); FillCollection(collection, property); continue; } if (property.IsFactoryType(property.PropertyType)) { uint id = _reader.ReadUInt32(); Type tSpec = ObjectFactory.getTypeFrom(id); if (tSpec != null) property.SetValue(obj, CreateObjectFromType(tSpec), null); else property.SetValue(obj, null, null); } else { property.SetValue(obj, CreateObjectFromType(property.PropertyType), null); } } } } #endregion #region Write object to data Main protected override bool SaveObjectToFileImplementation(object obj, string file) { using (_writer = new BigEndianBinaryWriter(File.Open(file, FileMode.Create))) { if (_writer == null) return false; WriteEngineVersion(); WriteObject(obj); } return true; } public byte[] SaveObjectToBuffer(object obj) { MemoryStream memStream = new MemoryStream(); Serialize(memStream, obj); return memStream.ToArray(); } public void Serialize(Stream stream, object obj) { _writer = new BigEndianBinaryWriter(stream); WriteEngineVersion(); if (obj is IUAFCollection) WriteCollection((IUAFCollection)obj, null); else WriteObject(obj); } #endregion #region Write object to data Internal private void WriteBaseTypeObject(object obj, Type objType) { CheckWrite(objType); if (objType.IsEnum) { _writer.Write((int)obj); return; } TypeConverter conv = TypeDescriptor.GetConverter(objType); switch (objType.Name) { case "Boolean": if ((Boolean)obj) _writer.Write((int)1); else _writer.Write((int)0); return; case "int": case "Int32": _writer.Write((int)obj); return; case "uint": case "UInt32": _writer.Write((uint)obj); return; case "float": case "Single": _writer.Write((float)obj); return; case "Double": _writer.Write((double)obj); return; case "ObjectPath": WriteComplexTypeObject(obj); return; case "String": { string pathStr = ((string)obj) ?? ""; var bytes = Encoding.UTF8.GetBytes(pathStr); _writer.Write((uint)bytes.Length); _writer.Write(bytes, 0, bytes.Length); } return; case "String8": { string pathStr = ((String8)obj).ToString(); var bytes = Encoding.UTF8.GetBytes(pathStr); _writer.Write((uint)bytes.Length); _writer.Write(bytes, 0, bytes.Length); } return; case "Path": { string pathStr = ((Path)obj).ToString(); string baseStr = ""; pathStr = pathStr.Replace('\\', '/'); int index = pathStr.LastIndexOf('/'); if (index > 0) { baseStr = pathStr.Substring(index + 1); pathStr = pathStr.Substring(0, index + 1); } _writer.Write((uint)baseStr.Length); _writer.Write(baseStr.ToCharArray(), 0, baseStr.Length); _writer.Write((uint)pathStr.Length); _writer.Write(pathStr.ToCharArray(), 0, pathStr.Length); _writer.Write((new StringID(((Path)obj).ToString())).getValue()); // stringID _writer.Write((uint)0); // flag } return; case "ColorInteger": case "LocalisationId": _writer.Write(int.Parse(obj.ToString())); return; case "Color": { _writer.Write(((float)((Color)obj).B) * (1.0f / 255.0f)); _writer.Write(((float)((Color)obj).G) * (1.0f / 255.0f)); _writer.Write(((float)((Color)obj).R) * (1.0f / 255.0f)); _writer.Write(((float)((Color)obj).A) * (1.0f / 255.0f)); } return; case "Angle": case "Volume": if (conv != null && conv.CanConvertTo(typeof(float))) { float f = (float)conv.ConvertTo(obj, typeof(float)); _writer.Write(f); } return; case "StringID": if (IncludeSpecialFields) { string baseStr = obj.ToString(); _writer.Write(baseStr.Length); _writer.Write(baseStr.ToCharArray(), 0, baseStr.Length); } else if (SerializeStringIdAsString && obj != null && !string.IsNullOrEmpty(obj.ToString())) { string baseStr = obj.ToString(); _writer.Write(StringID.FullStringTag); var bytes = Encoding.UTF8.GetBytes(baseStr); _writer.Write((uint)bytes.Length); _writer.Write(bytes, 0, bytes.Length); } else if (conv != null && conv.CanConvertTo(typeof(uint))) { uint ui = (uint)conv.ConvertTo(obj, typeof(uint)); _writer.Write(ui); } return; case "Vec2d": { var vector = (Vec2d)obj; _writer.Write(vector.X); _writer.Write(vector.Y); return; } case "Vec3d": { var vector = (Vec3d)obj; _writer.Write(vector.X); _writer.Write(vector.Y); _writer.Write(vector.Z); return; } case "Platform": { var platform = (Platform)obj; _writer.Write((uint)platform.Id); return; } } } private void WriteComplexTypeObject(object obj) { foreach (PropertyInfo property in SerializableFields(obj.GetType())) { object objIn = property.GetValue(obj, null); if (property.IsOptional()) { if (objIn == null) { _writer.Write(0); continue; } else { _writer.Write(1); } } if (BasicTypeList.IsBasicType(property.PropertyType)) { WriteBaseTypeObject(objIn, property.PropertyType); continue; } if (IsCollection(property.PropertyType)) { IUAFCollection lst = (IUAFCollection)(property.GetValue(obj, null)); WriteCollection(lst, property); continue; } if (property.IsFactoryType(property.PropertyType)) { if (objIn == null) { _writer.Write(uint.MaxValue); } else { uint id = ObjectFactory.getStringIDFrom(objIn.GetType()); _writer.Write(id); WriteObject(objIn); } } else { WriteObject(objIn); } } } private void WriteObject(object obj) { Type objType = TypeMap.ToSerializationType(obj.GetType()); if (BasicTypeList.IsBasicType(objType)) { WriteBaseTypeObject(obj, objType); } else { WriteComplexTypeObject(obj); } } private void WriteCollection(IUAFCollection collection, PropertyInfo property) { bool factoryType = property.IsFactoryType(collection.ElementType); uint nbElements = (uint)collection.Count; CheckWrite(EngineType.SizeT); _writer.Write(nbElements); // We create a copy of the collection, in order no to crash while enumerating it during the writing. var collectionCopy = collection.SerializableElements.Cast().ToArray(); foreach (var element in collectionCopy) { if (factoryType) { if (element == null) { _writer.Write(uint.MaxValue); } else { StringID id = ObjectFactory.getStringIDFrom(element.GetType()); WriteBaseTypeObject(id, typeof(StringID)); WriteObject(element); } } else { WriteObject(element); } } } #endregion #region Check Types public enum EngineType { Undefined, Unsigned_long, Char, SByte, Int16, Int32, UInt16, UInt32, UInt64, Color, ColorInteger, Angle, Boolean, Single, Double, String, StringID, Vec2d, Vec3d, Matrix44, Path, Volume, SizeT, ObjectPath, LocalisationId, Platform, } [Conditional("Diagnose")] private void CheckRead(Type expectedType, PropertyInfo targetProperty = null) { CheckRead(ToEngineType(expectedType), targetProperty); } [Conditional("Diagnose")] private void CheckRead(EngineType expectedType, PropertyInfo targetProperty = null) { var readType = (EngineType)_reader.ReadUInt32(); if (expectedType != readType) { object readValue = null; try { Type runtimeType = Type.GetType("System." + readType) ?? Type.GetType("JD.Serialization." + readType); if (runtimeType != null) { readValue = CreateBaseTypeObject(runtimeType, targetProperty, checkReadType: false); } } catch { } throw new Exception($"Property : {targetProperty}. Expected to read a {expectedType}. Received a {readType} ({readValue}). Details : {m_trace}"); } } [Conditional("Diagnose")] private void CheckWrite(Type type) { CheckWrite(ToEngineType(type)); } [Conditional("Diagnose")] private void CheckWrite(EngineType type) { _writer.Write((uint)type); } private static EngineType ToEngineType(Type type) { if (type.IsEnum) { type = typeof(int); } // Booleans are serialized as int32 in the engine. else if (type == typeof(bool)) { type = typeof(int); } return (EngineType)Enum.Parse(typeof(EngineType), type.Name); } /// /// Allows to diagnose what was read/what was written. /// private class DebugTrace : IDisposable { private readonly StringBuilder _trace = new StringBuilder(); private readonly LuaWriter _writer; private int maxIndex = 0; public DebugTrace() { _writer = new LuaWriter(new StringWriter(_trace)); } public IDisposable StartObject(object value) { IDisposable result = null; #if Diagnose string description = value == null ? null : value.ToString(); if (value != null && description != value.GetType().FullName) { description = string.Format("{0} ('{1}')", value.GetType().FullName, description); } _writer.WriteLine(description); result = _writer.Indent(); #endif return result; } public IDisposable StartList(PropertyInfo property) { IDisposable result = null; #if Diagnose _writer.WriteLine("List '{0}'", property == null ? "" : property.Name); result = _writer.Indent(); #endif return result; } [Conditional("Diagnose")] public void Property(object property, object propertyValue) { if (property is int) maxIndex = Math.Max((int)property, maxIndex); _writer.WriteLine("{0} = {1}", property, propertyValue); } [Conditional("Diagnose")] public void Print() { string file = System.IO.Path.GetTempFileName() + ".txt"; File.WriteAllText(file, _trace.ToString()); Process.Start(file); } public override string ToString() { return _trace.ToString(); } public void Dispose() { _writer?.Dispose(); } } #endregion public void Dispose() { _reader?.Dispose(); _writer?.Dispose(); } } }