Added Unrecoverable attribute.

Added ExternalStorageAttribute.
Moved external storage management into provider

InABox.Core/Attributes/ExternalStorageAttribute.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace InABox.Core
+{
+    /// <summary>
+    /// Indicates that this property is one that should, if necessary, be stored externally; for example, it may be a BLOB-type data, and would
+    /// thus be not suitable for database storage; perhaps it should be a file.
+    /// </summary>
+    /// <remarks>
+    /// The exact implementation of this property is specific to the SQL provider; it may not do anything.
+    /// <br/>
+    /// Any property marked with <see cref="ExternalStorageAttribute"/> should be a <see cref="byte[]"/>.
+    /// </remarks>
+    [AttributeUsage(AttributeTargets.Property)]
+    public class ExternalStorageAttribute : Attribute
+    {
+    }
+}

InABox.Core/Attributes/UnrecoverableAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace InABox.Core
+{
+    /// <summary>
+    /// Indicates that when deleting this <see cref="Entity"/>, it is not saved as a <see cref="Deletion"/>, but completely purged,
+    /// and thus is unrecoverable.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Class)]
+    public class UnrecoverableAttribute : Attribute
+    {
+    }
+}

InABox.Core/Classes/Document/Document.cs

@@ -27,6 +27,7 @@ namespace InABox.Core
         /// and neither can it be filtered on.
         /// </summary>
         [NullEditor]
+        [ExternalStorage]
         public byte[] Data { get; set; } = Array.Empty<byte>();
 
         /*[DoNotSerialize]

InABox.Core/Security/GlobalSecurityToken.cs

@@ -4,6 +4,7 @@ namespace InABox.Core
 {
     [UserTracking(typeof(User))]
     [Caption("Security Defaults")]
+    [Unrecoverable]
     public class GlobalSecurityToken : Entity, IPersistent, IRemotable, ILicense<CoreLicense>
     {
         [ComboLookupEditor(typeof(SecurityRestrictionGenerator))]

InABox.Core/Security/SecurityToken.cs

@@ -4,6 +4,7 @@ namespace InABox.Core
 {
     [UserTracking(typeof(User))]
     [Caption("Security Defaults")]
+    [Unrecoverable]
     public class SecurityToken : Entity, IPersistent, IRemotable, ILicense<CoreLicense>
     {
         [NullEditor]

InABox.Core/Security/UserSecurityToken.cs

@@ -4,6 +4,7 @@ namespace InABox.Core
 {
     [UserTracking(typeof(User))]
     [Caption("Security Overrides")]
+    [Unrecoverable]
     public class UserSecurityToken : Entity, IPersistent, IRemotable, ILicense<CoreLicense>
     {
         [NullEditor]

InABox.Core/User/Login.cs

@@ -3,6 +3,7 @@
 namespace InABox.Core
 {
     [UserTracking(false)]
+    [Unrecoverable]
     public class Login : Entity, IRemotable, IPersistent, ILicense<CoreLicense>
     {
         public UserLink User { get; set; }

InABox.Database/Stores/DocumentStore.cs

@@ -1,115 +0,0 @@
-using InABox.Core;
-using NPOI.POIFS.FileSystem;
-
-namespace InABox.Database;
-
-public class DocumentStore : Store<Document>
-{
-    private static readonly Column<Document> DataColumn = new(x => x.Data);
-
-    protected override CoreTable OnQuery(Filter<Document>? filter, Columns<Document>? columns, SortOrder<Document>? sort)
-    {
-        var getData = false;
-        if (columns is null || columns.Contains(DataColumn))
-        {
-            getData = true;
-            columns?.Add(x => x.ID);
-        }
-
-        var result = base.OnQuery(filter, columns, sort);
-
-        if (getData)
-        {
-            foreach(var row in result.Rows)
-            {
-                var id = row.Get<Document, Guid>(x => x.ID);
-                var filename = FileName(id);
-                if (File.Exists(filename))
-                {
-                    var data = File.ReadAllBytes(filename);
-                    row.Set<Document, byte[]>(x => x.Data, data);
-                }
-            }
-        }
-        return result;
-    }
-
-    protected override void OnSave(Document entity, ref string auditnote)
-    {
-        byte[]? data = null;
-        if(entity.Data != null && entity.Data.Length > 0)
-        {
-            data = entity.Data;
-            entity.Data = Array.Empty<byte>();
-        }
-        base.OnSave(entity, ref auditnote);
-
-        if(data is not null)
-        {
-            // Set data back so that it's not obvious what we've done.
-            entity.Data = data;
-            EnsureDirectory();
-            SaveData(entity.ID, data);
-        }
-    }
-
-    protected override void OnSave(IEnumerable<Document> entities, ref string auditnote)
-    {
-        var entityList = entities.AsIList();
-        var dataList = entityList.Select(x =>
-        {
-            byte[]? data = null;
-            if(x.Data != null && x.Data.Length > 0)
-            {
-                data = x.Data;
-                x.Data = Array.Empty<byte>();
-            }
-            return (data, x);
-        }).ToList();
-
-        base.OnSave(entityList, ref auditnote);
-
-        EnsureDirectory();
-        foreach (var item in dataList)
-        {
-            if(item.data is not null)
-            {
-                // Set data back so that it's not obvious what we've done.
-                item.x.Data = item.data;
-                SaveData(item.x.ID, item.data);
-            }
-        }
-    }
-
-    protected override void OnDelete(Document entity)
-    {
-        base.OnDelete(entity);
-    }
-
-    protected override void OnDelete(IEnumerable<Document> entities)
-    {
-        base.OnDelete(entities);
-    }
-
-    private void SaveData(Guid documentID, byte[] data)
-    {
-        File.WriteAllBytes(FileName(documentID), data);
-    }
-
-    private string Directory()
-    {
-        var directory = Path.GetDirectoryName(Provider.URL);
-        var filename = Path.GetFileName(Provider.URL);
-        return Path.Combine(directory ?? "", $"{filename}.data", nameof(Document));
-    }
-
-    private string FileName(Guid documentID)
-    {
-        return Path.Combine(Directory(), documentID.ToString());
-    }
-
-    private void EnsureDirectory()
-    {
-        System.IO.Directory.CreateDirectory(Directory());
-    }
-}

InABox.Database/Stores/GlobalSecurityTokenStore.cs

@@ -1,22 +0,0 @@
-using InABox.Core;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class GlobalSecurityTokenStore : Store<GlobalSecurityToken>
-    {
-        protected override void OnDelete(GlobalSecurityToken entity)
-        {
-            Provider.Purge(entity);
-        }
-
-        protected override void OnDelete(IEnumerable<GlobalSecurityToken> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

InABox.Database/Stores/LoginStore.cs

@@ -1,23 +0,0 @@
-using InABox.Core;
-using NPOI.POIFS.FileSystem;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class LoginStore : Store<Login>
-    {
-        protected override void OnDelete(Login entity)
-        {
-            Provider.Purge(entity);
-        }
-
-        protected override void OnDelete(IEnumerable<Login> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

InABox.Database/Stores/SecurityTokenStore.cs

@@ -1,22 +0,0 @@
-using InABox.Core;
-using System;
-using System.Collections.Generic;
-using System.Configuration;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class SecurityTokenStore : Store<SecurityToken>
-    {
-        protected override void OnDelete(SecurityToken entity)
-        {
-            Provider.Purge(entity);
-        }
-        protected override void OnDelete(IEnumerable<SecurityToken> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

InABox.Database/Stores/Store.cs

@@ -154,10 +154,10 @@ namespace InABox.Database
         private IEnumerable<T> RunScript(ScriptType type, IEnumerable<T> entities)
         {
             var scriptname = type.ToString();
-            var variable = typeof(T).EntityName().Split('.').Last() + "s";
             var key = string.Format("{0} {1}", typeof(T).EntityName(), scriptname);
             if (DbFactory.LoadedScripts.ContainsKey(key))
             {
+                var variable = typeof(T).EntityName().Split('.').Last() + "s";
                 var script = DbFactory.LoadedScripts[key];
                 script.SetValue("Store", this);
                 script.SetValue(variable, entities);

InABox.Database/Stores/UserSecurityTokenStore.cs

@@ -1,22 +0,0 @@
-using InABox.Core;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class UserSecurityTokenStore : Store<UserSecurityToken>
-    {
-        protected override void OnDelete(UserSecurityToken entity)
-        {
-            Provider.Purge(entity);
-        }
-
-        protected override void OnDelete(IEnumerable<UserSecurityToken> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

inabox.database.sqlite/SQLiteProvider.cs

@@ -2383,9 +2383,21 @@ namespace InABox.Database.SQLite
             //LogReset();
 
             //LogStart();
-            var cols = CoreUtils.GetColumns(T, columns);
             //LogStop("GetColumns");
 
+            var cols = CoreUtils.GetColumns(T, columns);
+
+            var blobColumns = Columns.Create(T);
+            foreach(var column in cols.GetColumns())
+            {
+                var prop = DatabaseSchema.Property(T, column.Property);
+                if (prop is not null && prop.HasAttribute<ExternalStorageAttribute>())
+                {
+                    blobColumns.Add(column);
+                    cols.Add("ID");
+                }
+            }
+
             var result = new CoreTable(T.EntityName());
             foreach (var col in cols.GetColumns())
                 result.Columns.Add(new CoreColumn { ColumnName = col.Property, DataType = col.Type });
@@ -2508,6 +2520,19 @@ namespace InABox.Database.SQLite
                 }
             }
 
+            foreach(var column in blobColumns.ColumnNames())
+            {
+                foreach(var row in result.Rows)
+                {
+                    var id = row.Get<Guid>("ID");
+                    var data = GetExternalData(T, column, id);
+                    if(data is not null)
+                    {
+                        row.Set<byte[]>(column, data);
+                    }
+                }
+            }
+
             if (log)
             {
                 var duration = DateTime.Now - start;
@@ -2609,38 +2634,110 @@ namespace InABox.Database.SQLite
 
         #region Save
 
+        private static readonly Dictionary<Type, IProperty[]> _externalProperties = new Dictionary<Type, IProperty[]>();
+        private static IProperty[] GetExternalProperties(Type T)
+        {
+            if(!_externalProperties.TryGetValue(T, out var properties))
+            {
+                properties = DatabaseSchema.Properties(T).Where(x => x.HasAttribute<ExternalStorageAttribute>()).ToArray();
+                _externalProperties.Add(T, properties);
+            }
+            return properties;
+        }
+
+        private void OnSaveNonGeneric(Type T, Entity entity)
+        {
+            var props = GetExternalProperties(T);
+            List<(IProperty, byte[])>? data = null;
+            if (props.Any())
+            {
+                data = new List<(IProperty, byte[])>();
+                foreach(var prop in props)
+                {
+                    var value = prop.Getter()(entity) as byte[];
+                    if(value is not null && value.Length > 0)
+                    {
+                        data.Add(new(prop, value));
+                        prop.Setter()(entity, Array.Empty<byte>());
+                    }
+                }
+            }
+
+            using var access = GetWriteAccess();
+            using var command = access.CreateCommand();
+
+            PrepareUpsertNonGeneric(T, command, entity);
+            command.ExecuteNonQuery();
+
+            if(data is not null)
+            {
+                foreach(var (prop, value) in data)
+                {
+                    SaveExternalData(T, prop.Name, entity.ID, value);
+                    prop.Setter()(entity, value);
+                }
+            }
+        }
+
         private void OnSaveNonGeneric(Type T, IEnumerable<Entity> entities)
         {
+            // Casting to IList so that we can use it multiple times.
+            entities = entities.AsIList();
+
             if (!entities.Any())
                 return;
 
-            using (var access = GetWriteAccess())
+            var props = GetExternalProperties(T);
+            List<(IProperty, List<(Entity, byte[])>)>? data = null;
+            if (props.Any())
             {
-                Exception? error = null;
-                using (var transaction = access.BeginTransaction())
+                data = new List<(IProperty, List<(Entity, byte[])>)>();
+                foreach(var prop in props)
                 {
-                    try
+                    var lst = new List<(Entity, byte[])>();
+                    foreach(var entity in entities)
                     {
-                        using (var command = access.CreateCommand())
+                        var value = prop.Getter()(entity) as byte[];
+                        if(value is not null && value.Length > 0)
                         {
-                            foreach (var entity in entities)
-                            {
-                                PrepareUpsertNonGeneric(T, command, entity);
-                                command.ExecuteNonQuery();
-                            }
-
-                            transaction.Commit();
+                            lst.Add((entity, value));
+                            prop.Setter()(entity, Array.Empty<byte>());
                         }
                     }
-                    catch (Exception e)
+                    data.Add(new(prop, lst));
+                }
+            }
+
+            using var access = GetWriteAccess();
+            using var transaction = access.BeginTransaction();
+
+            try
+            {
+                using var command = access.CreateCommand();
+                foreach (var entity in entities)
+                {
+                    PrepareUpsertNonGeneric(T, command, entity);
+                    command.ExecuteNonQuery();
+                }
+
+                transaction.Commit();
+
+                if(data is not null)
+                {
+                    foreach(var (property, list) in data)
                     {
-                        error = e;
-                        transaction.Rollback();
+                        foreach(var (entity, value) in list)
+                        {
+                            SaveExternalData(T, property.Name, entity.ID, value);
+                            property.Setter()(entity, value);
+                        }
                     }
                 }
-
-                if (error != null)
-                    throw error;
+            }
+            catch (Exception)
+            {
+                transaction.Rollback();
+                throw;
             }
         }
         private void OnSave<T>(IEnumerable<T> entities) where T : Entity
@@ -2673,28 +2770,6 @@ namespace InABox.Database.SQLite
             }
             OnSave(entities);
         }
-        private void OnSaveNonGeneric(Type T, Entity entity)
-        {
-            Exception? error = null;
-            using (var access = GetWriteAccess())
-            {
-                using (var command = access.CreateCommand())
-                {
-                    try
-                    {
-                        PrepareUpsertNonGeneric(T, command, entity);
-                        command.ExecuteNonQuery();
-                    }
-                    catch (Exception e)
-                    {
-                        error = e;
-                    }
-                }
-            }
-
-            if (error != null)
-                throw error;
-        }
 
         private void OnSave<T>(T entity) where T : Entity
             => OnSaveNonGeneric(typeof(T), entity);
@@ -2714,49 +2789,40 @@ namespace InABox.Database.SQLite
 
         public void Purge<T>(T entity) where T : Entity
         {
-            using (var access = GetWriteAccess())
-            {
-                using (var command = access.CreateCommand())
-                {
-                    PrepareDelete(command, entity);
-                    var rows = command.ExecuteNonQuery();
-                }
-            }
+            using var access = GetWriteAccess();
+            using var command = access.CreateCommand();
+            PrepareDelete(command, entity);
+            var rows = command.ExecuteNonQuery();
         }
 
         public void Purge<T>(IEnumerable<T> entities) where T : Entity
         {
+            // Casting to IList so that we can use it multiple times.
+            entities = entities.AsIList();
+
             if (!entities.Any())
                 return;
 
-            Exception? error = null;
-            using (var access = GetWriteAccess())
+            using var access = GetWriteAccess();
+            using var transaction = access.BeginTransaction();
+            try
             {
-                using (var transaction = access.BeginTransaction())
+                using (var command = access.CreateCommand())
                 {
-                    try
+                    foreach (var entity in entities)
                     {
-                        using (var command = access.CreateCommand())
-                        {
-                            foreach (var entity in entities)
-                            {
-                                PrepareDelete(command, entity);
-                                var rows = command.ExecuteNonQuery();
-                            }
-                        }
-
-                        transaction.Commit();
-                    }
-                    catch (Exception e)
-                    {
-                        transaction.Rollback();
-                        error = e;
+                        PrepareDelete(command, entity);
+                        var rows = command.ExecuteNonQuery();
                     }
                 }
-            }
 
-            if (error != null)
-                throw error;
+                transaction.Commit();
+            }
+            catch (Exception)
+            {
+                transaction.Rollback();
+                throw;
+            }
         }
 
         private Dictionary<Type, List<Tuple<Type, string>>> _cascades = new();
@@ -2898,30 +2964,43 @@ namespace InABox.Database.SQLite
             {
                 return;
             }
-            entity = DoQuery(
-                new Filter<T>(x => x.ID).IsEqualTo(entity.ID),
-                DeletionData.DeletionColumns<T>(),
-                null,
-                int.MaxValue,
-                false,
-                false
-            ).Rows.First().ToObject<T>();
-
-            var deletionData = new DeletionData();
-            deletionData.DeleteEntity(entity);
-            CascadeDelete(typeof(T), new Guid[] { entity.ID }, deletionData);
-
-            var tableName = typeof(T).Name;
-            var deletion = new Deletion()
-            {
-                DeletionDate = DateTime.Now,
-                HeadTable = tableName,
-                Description = entity.ToString() ?? "",
-                DeletedBy = userID,
-                Data = Serialization.Serialize(deletionData)
-            };
-            OnSave(deletion);
-            Purge(entity);
+            if (typeof(T).HasAttribute<UnrecoverableAttribute>())
+            {
+                Purge(entity);
+
+                var props = GetExternalProperties(typeof(T));
+                foreach(var prop in props)
+                {
+                    DeleteExternalData(typeof(T), prop.Name, entity.ID);
+                }
+            }
+            else
+            {
+                entity = DoQuery(
+                    new Filter<T>(x => x.ID).IsEqualTo(entity.ID),
+                    DeletionData.DeletionColumns<T>(),
+                    null,
+                    int.MaxValue,
+                    false,
+                    false
+                ).Rows.First().ToObject<T>();
+
+                var deletionData = new DeletionData();
+                deletionData.DeleteEntity(entity);
+                CascadeDelete(typeof(T), new Guid[] { entity.ID }, deletionData);
+
+                var tableName = typeof(T).Name;
+                var deletion = new Deletion()
+                {
+                    DeletionDate = DateTime.Now,
+                    HeadTable = tableName,
+                    Description = entity.ToString() ?? "",
+                    DeletedBy = userID,
+                    Data = Serialization.Serialize(deletionData)
+                };
+                OnSave(deletion);
+                Purge(entity);
+            }
         }
 
         public void Delete<T>(IEnumerable<T> entities, string userID) where T : Entity, new()
@@ -2930,33 +3009,51 @@ namespace InABox.Database.SQLite
             {
                 return;
             }
+            entities = entities.AsIList();
             if (!entities.Any())
                 return;
-            var ids = entities.Select(x => x.ID).ToArray();
-            var entityList = Query(
-                new Filter<T>(x => x.ID).InList(ids),
-                DeletionData.DeletionColumns<T>()).Rows.Select(x => x.ToObject<T>()).ToList();
-            if (!entityList.Any())
-                return;
 
-            var deletionData = new DeletionData();
-            foreach (var entity in entityList)
+            if (typeof(T).HasAttribute<UnrecoverableAttribute>())
             {
-                deletionData.DeleteEntity(entity);
-            }
-            CascadeDelete(typeof(T), ids, deletionData);
+                Purge(entities);
 
-            var tableName = typeof(T).Name;
-            var deletion = new Deletion()
+                var props = GetExternalProperties(typeof(T));
+                foreach(var prop in props)
+                {
+                    foreach(var entity in entities)
+                    {
+                        DeleteExternalData(typeof(T), prop.Name, entity.ID);
+                    }
+                }
+            }
+            else
             {
-                DeletionDate = DateTime.Now,
-                HeadTable = tableName,
-                Description = $"Deleted {entityList.Count} entries",
-                DeletedBy = userID,
-                Data = Serialization.Serialize(deletionData)
-            };
-            OnSave(deletion);
-            Purge(entities);
+                var ids = entities.Select(x => x.ID).ToArray();
+                var entityList = Query(
+                    new Filter<T>(x => x.ID).InList(ids),
+                    DeletionData.DeletionColumns<T>()).Rows.Select(x => x.ToObject<T>()).ToList();
+                if (!entityList.Any())
+                    return;
+
+                var deletionData = new DeletionData();
+                foreach (var entity in entityList)
+                {
+                    deletionData.DeleteEntity(entity);
+                }
+                CascadeDelete(typeof(T), ids, deletionData);
+
+                var tableName = typeof(T).Name;
+                var deletion = new Deletion()
+                {
+                    DeletionDate = DateTime.Now,
+                    HeadTable = tableName,
+                    Description = $"Deleted {entityList.Count} entries",
+                    DeletedBy = userID,
+                    Data = Serialization.Serialize(deletionData)
+                };
+                OnSave(deletion);
+                Purge(entities);
+            }
         }
 
         private void AddDeletionType(Type type, List<Type> deletions)
@@ -2985,6 +3082,24 @@ namespace InABox.Database.SQLite
         
         public void Purge(Deletion deletion)
         {
+            var data = Serialization.Deserialize<DeletionData>(deletion.Data);
+            if(data is not null)
+            {
+                foreach(var (entityName, cascade) in data.Cascades)
+                {
+                    if (!CoreUtils.TryGetEntity(entityName, out var entityType)) continue;
+
+                    var props = GetExternalProperties(entityType);
+                    foreach(var prop in props)
+                    {
+                        foreach(var entity in cascade.ToObjects(entityType).Cast<Entity>())
+                        {
+                            DeleteExternalData(entityType, prop.Name, entity.ID);
+                        }
+                    }
+                }
+            }
+
             Purge<Deletion>(deletion);
         }
 
@@ -3050,5 +3165,52 @@ namespace InABox.Database.SQLite
 
         #endregion
 
+        #region External Data Storage
+
+        private string ExternalDataFolder(Type T, string columnName, string idString)
+        {
+            return Path.Combine(
+                Path.GetDirectoryName(URL) ?? "",
+                $"{Path.GetFileName(URL)}.data",
+                T.Name,
+                columnName,
+                idString.Substring(0, 2));
+        }
+
+        private byte[]? GetExternalData(Type T, string columnName, Guid id)
+        {
+            var idString = id.ToString();
+            var filename = Path.Combine(ExternalDataFolder(T, columnName, idString), idString);
+            try
+            {
+                return File.ReadAllBytes(filename);
+            }
+            catch(Exception e)
+            {
+                Logger.Send(LogType.Error, "", $"Could not load external {T.Name}.{columnName}: {e.Message}");
+                return null;
+            }
+        }
+        private void SaveExternalData(Type T, string columnName, Guid id, byte[] data)
+        {
+            var idString = id.ToString();
+            var directory = ExternalDataFolder(T, columnName, idString);
+            Directory.CreateDirectory(directory);
+            var filename = Path.Combine(directory, idString);
+            File.WriteAllBytes(filename, data);
+        }
+        private void DeleteExternalData(Type T, string columnName, Guid id)
+        {
+            var idString = id.ToString();
+            var directory = ExternalDataFolder(T, columnName, idString);
+            Directory.CreateDirectory(directory);
+            var filename = Path.Combine(directory, idString);
+            if (File.Exists(filename))
+            {
+                File.Delete(filename);
+            }
+        }
+
+        #endregion
     }
 }

inabox.wpf/DynamicGrid/DynamicOneToManyGrid.cs

@@ -52,7 +52,7 @@ namespace InABox.DynamicGrid
 
             options.Add(DynamicGridOption.RecordCount)
                 .Add(DynamicGridOption.SelectColumns);
-
+            
             if (Security.CanEdit<TMany>() && !ReadOnly)
                 options.Add(DynamicGridOption.AddRows).Add(DynamicGridOption.EditRows);
             if (Security.CanDelete<TMany>() && !ReadOnly)