Merge remote-tracking branch 'origin/kenric' into frank

InABox.Core/Attributes/ExternalStorageAttribute.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace InABox.Core
+{
+    /// <summary>
+    /// Indicates that this property is one that should, if necessary, be stored externally; for example, it may be a BLOB-type data, and would
+    /// thus be not suitable for database storage; perhaps it should be a file.
+    /// </summary>
+    /// <remarks>
+    /// The exact implementation of this property is specific to the SQL provider; it may not do anything.
+    /// <br/>
+    /// Any property marked with <see cref="ExternalStorageAttribute"/> should be a <see cref="byte[]"/>.
+    /// </remarks>
+    [AttributeUsage(AttributeTargets.Property)]
+    public class ExternalStorageAttribute : Attribute
+    {
+    }
+}

InABox.Core/Attributes/UnrecoverableAttribute.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace InABox.Core
+{
+    /// <summary>
+    /// Indicates that when deleting this <see cref="Entity"/>, it is not saved as a <see cref="Deletion"/>, but completely purged,
+    /// and thus is unrecoverable.
+    /// </summary>
+    [AttributeUsage(AttributeTargets.Class)]
+    public class UnrecoverableAttribute : Attribute
+    {
+    }
+}

InABox.Core/Classes/Document/Document.cs

@@ -22,7 +22,12 @@ namespace InABox.Core
         [CodeEditor(Editable = Editable.Enabled)]
         public string CRC { get; set; } = "";
 
+        /// <summary>
+        /// This is a special property, and is not stored in the database. Hence, it <b>cannot</b> be available to <see cref="DocumentLink"/>,
+        /// and neither can it be filtered on.
+        /// </summary>
         [NullEditor]
+        [ExternalStorage]
         public byte[] Data { get; set; } = Array.Empty<byte>();
 
         /*[DoNotSerialize]

InABox.Core/Client/BaseClient.cs

@@ -281,7 +281,14 @@ namespace InABox.Clients
 
         public Dictionary<string, CoreTable> QueryMultiple(Dictionary<string, IQueryDef> queries)
         {
-            return DoQueryMultiple(queries);
+            if(queries.Count == 0)
+            {
+                return new Dictionary<string, CoreTable>();
+            }
+            else
+            {
+                return DoQueryMultiple(queries);
+            }
         }
 
         protected abstract Dictionary<string, CoreTable> DoQueryMultiple(Dictionary<string, IQueryDef> queries);
@@ -294,7 +301,7 @@ namespace InABox.Clients
                 Dictionary<string, CoreTable>? result = null;
                 try
                 {
-                    result = DoQueryMultiple(queries);
+                    result = QueryMultiple(queries);
                 }
                 catch (Exception e)
                 {

InABox.Core/Client/Client.cs

@@ -36,6 +36,8 @@ namespace InABox.Clients
             => Results[typeof(T).Name].ToObjects<T>();
 
         public CoreTable Get(string name) => Results[name];
+
+        public CoreTable GetOrDefault(string name) => Results.GetValueOrDefault(name);
     }
 
     public abstract class Client
@@ -519,11 +521,18 @@ namespace InABox.Clients
                 CheckSupported();
                 var items = entities.AsArray();
                 if (items.Any())
+                {
                     _client.Save(items, auditnote, (i, e) =>
                     {
                         timer.Dispose(i.Count());
                         callback?.Invoke(i, e);
                     });
+                }
+                else
+                {
+                    timer.Dispose(0);
+                    callback?.Invoke(items, null);
+                }
             }
             catch (RequestException e)
             {

InABox.Core/MultiQuery/QueryDef.cs

@@ -37,6 +37,19 @@ namespace InABox.Clients
         public ISortOrder? SortOrder { get; }
     }
 
+    public class KeyedQueryDef : QueryDef, IKeyedQueryDef
+    {
+        public string Key { get; }
+
+        public KeyedQueryDef(string key, Type type, IFilter? filter = null, IColumns? columns = null, ISortOrder? sortOrder = null) : base(type)
+        {
+            Key = key;
+            Filter = filter;
+            Columns = columns;
+            SortOrder = sortOrder;
+        }
+    }
+
     public class KeyedQueryDef<T> : QueryDef<T>, IKeyedQueryDef where T : Entity, IRemotable, IPersistent, new()
     {
         public string Key { get; }

InABox.Core/Security/GlobalSecurityToken.cs

@@ -4,6 +4,7 @@ namespace InABox.Core
 {
     [UserTracking(typeof(User))]
     [Caption("Security Defaults")]
+    [Unrecoverable]
     public class GlobalSecurityToken : Entity, IPersistent, IRemotable, ILicense<CoreLicense>
     {
         [ComboLookupEditor(typeof(SecurityRestrictionGenerator))]

InABox.Core/Security/SecurityToken.cs

@@ -4,6 +4,7 @@ namespace InABox.Core
 {
     [UserTracking(typeof(User))]
     [Caption("Security Defaults")]
+    [Unrecoverable]
     public class SecurityToken : Entity, IPersistent, IRemotable, ILicense<CoreLicense>
     {
         [NullEditor]

InABox.Core/Security/UserSecurityToken.cs

@@ -4,6 +4,7 @@ namespace InABox.Core
 {
     [UserTracking(typeof(User))]
     [Caption("Security Overrides")]
+    [Unrecoverable]
     public class UserSecurityToken : Entity, IPersistent, IRemotable, ILicense<CoreLicense>
     {
         [NullEditor]

InABox.Core/User/Login.cs

@@ -3,6 +3,7 @@
 namespace InABox.Core
 {
     [UserTracking(false)]
+    [Unrecoverable]
     public class Login : Entity, IRemotable, IPersistent, ILicense<CoreLicense>
     {
         public UserLink User { get; set; }

InABox.Database/Stores/DocumentStore.cs

@@ -1,33 +0,0 @@
-using InABox.Core;
-
-namespace InABox.Database;
-
-public class DocumentStore : Store<Document>
-{
-    protected override void OnSave(Document entity, ref string auditnote)
-    {
-        //if (!Provider.IsRelational())
-        //{
-        //    byte[] data = entity.Data;
-        //    entity.Data = new byte[] { };
-        //    base.OnSave(entity);
-        //    entity.Data = data;
-        //    Provider.SaveFile(entity.ID, data);
-        //}
-        //else
-        base.OnSave(entity, ref auditnote);
-    }
-
-    protected override void OnSave(IEnumerable<Document> entities, ref string auditnote)
-    {
-        foreach (var entity in entities)
-            OnSave(entity, ref auditnote);
-    }
-
-    protected override void AfterDelete(Document entity)
-    {
-        base.AfterDelete(entity);
-        //if (!Provider.IsRelational())
-        //    Provider.DeleteFile(entity.ID);
-    }
-}

InABox.Database/Stores/GlobalSecurityTokenStore.cs

@@ -1,22 +0,0 @@
-using InABox.Core;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class GlobalSecurityTokenStore : Store<GlobalSecurityToken>
-    {
-        protected override void OnDelete(GlobalSecurityToken entity)
-        {
-            Provider.Purge(entity);
-        }
-
-        protected override void OnDelete(IEnumerable<GlobalSecurityToken> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

InABox.Database/Stores/LoginStore.cs

@@ -1,23 +0,0 @@
-using InABox.Core;
-using NPOI.POIFS.FileSystem;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class LoginStore : Store<Login>
-    {
-        protected override void OnDelete(Login entity)
-        {
-            Provider.Purge(entity);
-        }
-
-        protected override void OnDelete(IEnumerable<Login> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

InABox.Database/Stores/SecurityTokenStore.cs

@@ -1,22 +0,0 @@
-using InABox.Core;
-using System;
-using System.Collections.Generic;
-using System.Configuration;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class SecurityTokenStore : Store<SecurityToken>
-    {
-        protected override void OnDelete(SecurityToken entity)
-        {
-            Provider.Purge(entity);
-        }
-        protected override void OnDelete(IEnumerable<SecurityToken> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

InABox.Database/Stores/Store.cs

@@ -157,10 +157,10 @@ namespace InABox.Database
         private IEnumerable<T> RunScript(ScriptType type, IEnumerable<T> entities)
         {
             var scriptname = type.ToString();
-            var variable = typeof(T).EntityName().Split('.').Last() + "s";
             var key = string.Format("{0} {1}", typeof(T).EntityName(), scriptname);
             if (DbFactory.LoadedScripts.ContainsKey(key))
             {
+                var variable = typeof(T).EntityName().Split('.').Last() + "s";
                 var script = DbFactory.LoadedScripts[key];
                 script.SetValue("Store", this);
                 script.SetValue(variable, entities);

InABox.Database/Stores/UserSecurityTokenStore.cs

@@ -1,22 +0,0 @@
-using InABox.Core;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace InABox.Database
-{
-    public class UserSecurityTokenStore : Store<UserSecurityToken>
-    {
-        protected override void OnDelete(UserSecurityToken entity)
-        {
-            Provider.Purge(entity);
-        }
-
-        protected override void OnDelete(IEnumerable<UserSecurityToken> entities)
-        {
-            Provider.Purge(entities);
-        }
-    }
-}

inabox.database.sqlite/SQLiteProvider.cs

@@ -2388,9 +2388,21 @@ namespace InABox.Database.SQLite
             //LogReset();
 
             //LogStart();
-            var cols = CoreUtils.GetColumns(T, columns);
             //LogStop("GetColumns");
 
+            var cols = CoreUtils.GetColumns(T, columns);
+
+            var blobColumns = Columns.Create(T);
+            foreach(var column in cols.GetColumns())
+            {
+                var prop = DatabaseSchema.Property(T, column.Property);
+                if (prop is not null && prop.HasAttribute<ExternalStorageAttribute>())
+                {
+                    blobColumns.Add(column);
+                    cols.Add("ID");
+                }
+            }
+
             var result = new CoreTable(T.EntityName());
             foreach (var col in cols.GetColumns())
                 result.Columns.Add(new CoreColumn { ColumnName = col.Property, DataType = col.Type });
@@ -2513,6 +2525,19 @@ namespace InABox.Database.SQLite
                 }
             }
 
+            foreach(var column in blobColumns.ColumnNames())
+            {
+                foreach(var row in result.Rows)
+                {
+                    var id = row.Get<Guid>("ID");
+                    var data = GetExternalData(T, column, id);
+                    if(data is not null)
+                    {
+                        row.Set<byte[]>(column, data);
+                    }
+                }
+            }
+
             if (log)
             {
                 var duration = DateTime.Now - start;
@@ -2614,38 +2639,110 @@ namespace InABox.Database.SQLite
 
         #region Save
 
+        private static readonly Dictionary<Type, IProperty[]> _externalProperties = new Dictionary<Type, IProperty[]>();
+        private static IProperty[] GetExternalProperties(Type T)
+        {
+            if(!_externalProperties.TryGetValue(T, out var properties))
+            {
+                properties = DatabaseSchema.Properties(T).Where(x => x.HasAttribute<ExternalStorageAttribute>()).ToArray();
+                _externalProperties.Add(T, properties);
+            }
+            return properties;
+        }
+
+        private void OnSaveNonGeneric(Type T, Entity entity)
+        {
+            var props = GetExternalProperties(T);
+            List<(IProperty, byte[])>? data = null;
+            if (props.Any())
+            {
+                data = new List<(IProperty, byte[])>();
+                foreach(var prop in props)
+                {
+                    var value = prop.Getter()(entity) as byte[];
+                    if(value is not null && value.Length > 0)
+                    {
+                        data.Add(new(prop, value));
+                        prop.Setter()(entity, Array.Empty<byte>());
+                    }
+                }
+            }
+
+            using var access = GetWriteAccess();
+            using var command = access.CreateCommand();
+
+            PrepareUpsertNonGeneric(T, command, entity);
+            command.ExecuteNonQuery();
+
+            if(data is not null)
+            {
+                foreach(var (prop, value) in data)
+                {
+                    SaveExternalData(T, prop.Name, entity.ID, value);
+                    prop.Setter()(entity, value);
+                }
+            }
+        }
+
         private void OnSaveNonGeneric(Type T, IEnumerable<Entity> entities)
         {
+            // Casting to IList so that we can use it multiple times.
+            entities = entities.AsIList();
+
             if (!entities.Any())
                 return;
 
-            using (var access = GetWriteAccess())
+            var props = GetExternalProperties(T);
+            List<(IProperty, List<(Entity, byte[])>)>? data = null;
+            if (props.Any())
             {
-                Exception? error = null;
-                using (var transaction = access.BeginTransaction())
+                data = new List<(IProperty, List<(Entity, byte[])>)>();
+                foreach(var prop in props)
                 {
-                    try
+                    var lst = new List<(Entity, byte[])>();
+                    foreach(var entity in entities)
                     {
-                        using (var command = access.CreateCommand())
+                        var value = prop.Getter()(entity) as byte[];
+                        if(value is not null && value.Length > 0)
                         {
-                            foreach (var entity in entities)
-                            {
-                                PrepareUpsertNonGeneric(T, command, entity);
-                                command.ExecuteNonQuery();
-                            }
-
-                            transaction.Commit();
+                            lst.Add((entity, value));
+                            prop.Setter()(entity, Array.Empty<byte>());
                         }
                     }
-                    catch (Exception e)
+                    data.Add(new(prop, lst));
+                }
+            }
+
+            using var access = GetWriteAccess();
+            using var transaction = access.BeginTransaction();
+
+            try
+            {
+                using var command = access.CreateCommand();
+                foreach (var entity in entities)
+                {
+                    PrepareUpsertNonGeneric(T, command, entity);
+                    command.ExecuteNonQuery();
+                }
+
+                transaction.Commit();
+
+                if(data is not null)
+                {
+                    foreach(var (property, list) in data)
                     {
-                        error = e;
-                        transaction.Rollback();
+                        foreach(var (entity, value) in list)
+                        {
+                            SaveExternalData(T, property.Name, entity.ID, value);
+                            property.Setter()(entity, value);
+                        }
                     }
                 }
-
-                if (error != null)
-                    throw error;
+            }
+            catch (Exception)
+            {
+                transaction.Rollback();
+                throw;
             }
         }
         private void OnSave<T>(IEnumerable<T> entities) where T : Entity
@@ -2678,28 +2775,6 @@ namespace InABox.Database.SQLite
             }
             OnSave(entities);
         }
-        private void OnSaveNonGeneric(Type T, Entity entity)
-        {
-            Exception? error = null;
-            using (var access = GetWriteAccess())
-            {
-                using (var command = access.CreateCommand())
-                {
-                    try
-                    {
-                        PrepareUpsertNonGeneric(T, command, entity);
-                        command.ExecuteNonQuery();
-                    }
-                    catch (Exception e)
-                    {
-                        error = e;
-                    }
-                }
-            }
-
-            if (error != null)
-                throw error;
-        }
 
         private void OnSave<T>(T entity) where T : Entity
             => OnSaveNonGeneric(typeof(T), entity);
@@ -2719,49 +2794,40 @@ namespace InABox.Database.SQLite
 
         public void Purge<T>(T entity) where T : Entity
         {
-            using (var access = GetWriteAccess())
-            {
-                using (var command = access.CreateCommand())
-                {
-                    PrepareDelete(command, entity);
-                    var rows = command.ExecuteNonQuery();
-                }
-            }
+            using var access = GetWriteAccess();
+            using var command = access.CreateCommand();
+            PrepareDelete(command, entity);
+            var rows = command.ExecuteNonQuery();
         }
 
         public void Purge<T>(IEnumerable<T> entities) where T : Entity
         {
+            // Casting to IList so that we can use it multiple times.
+            entities = entities.AsIList();
+
             if (!entities.Any())
                 return;
 
-            Exception? error = null;
-            using (var access = GetWriteAccess())
+            using var access = GetWriteAccess();
+            using var transaction = access.BeginTransaction();
+            try
             {
-                using (var transaction = access.BeginTransaction())
+                using (var command = access.CreateCommand())
                 {
-                    try
+                    foreach (var entity in entities)
                     {
-                        using (var command = access.CreateCommand())
-                        {
-                            foreach (var entity in entities)
-                            {
-                                PrepareDelete(command, entity);
-                                var rows = command.ExecuteNonQuery();
-                            }
-                        }
-
-                        transaction.Commit();
-                    }
-                    catch (Exception e)
-                    {
-                        transaction.Rollback();
-                        error = e;
+                        PrepareDelete(command, entity);
+                        var rows = command.ExecuteNonQuery();
                     }
                 }
-            }
 
-            if (error != null)
-                throw error;
+                transaction.Commit();
+            }
+            catch (Exception)
+            {
+                transaction.Rollback();
+                throw;
+            }
         }
 
         private Dictionary<Type, List<Tuple<Type, string>>> _cascades = new();
@@ -2903,30 +2969,43 @@ namespace InABox.Database.SQLite
             {
                 return;
             }
-            entity = DoQuery(
-                new Filter<T>(x => x.ID).IsEqualTo(entity.ID),
-                DeletionData.DeletionColumns<T>(),
-                null,
-                int.MaxValue,
-                false,
-                false
-            ).Rows.First().ToObject<T>();
-
-            var deletionData = new DeletionData();
-            deletionData.DeleteEntity(entity);
-            CascadeDelete(typeof(T), new Guid[] { entity.ID }, deletionData);
-
-            var tableName = typeof(T).Name;
-            var deletion = new Deletion()
-            {
-                DeletionDate = DateTime.Now,
-                HeadTable = tableName,
-                Description = entity.ToString() ?? "",
-                DeletedBy = userID,
-                Data = Serialization.Serialize(deletionData)
-            };
-            OnSave(deletion);
-            Purge(entity);
+            if (typeof(T).HasAttribute<UnrecoverableAttribute>())
+            {
+                Purge(entity);
+
+                var props = GetExternalProperties(typeof(T));
+                foreach(var prop in props)
+                {
+                    DeleteExternalData(typeof(T), prop.Name, entity.ID);
+                }
+            }
+            else
+            {
+                entity = DoQuery(
+                    new Filter<T>(x => x.ID).IsEqualTo(entity.ID),
+                    DeletionData.DeletionColumns<T>(),
+                    null,
+                    int.MaxValue,
+                    false,
+                    false
+                ).Rows.First().ToObject<T>();
+
+                var deletionData = new DeletionData();
+                deletionData.DeleteEntity(entity);
+                CascadeDelete(typeof(T), new Guid[] { entity.ID }, deletionData);
+
+                var tableName = typeof(T).Name;
+                var deletion = new Deletion()
+                {
+                    DeletionDate = DateTime.Now,
+                    HeadTable = tableName,
+                    Description = entity.ToString() ?? "",
+                    DeletedBy = userID,
+                    Data = Serialization.Serialize(deletionData)
+                };
+                OnSave(deletion);
+                Purge(entity);
+            }
         }
 
         public void Delete<T>(IEnumerable<T> entities, string userID) where T : Entity, new()
@@ -2935,33 +3014,51 @@ namespace InABox.Database.SQLite
             {
                 return;
             }
+            entities = entities.AsIList();
             if (!entities.Any())
                 return;
-            var ids = entities.Select(x => x.ID).ToArray();
-            var entityList = Query(
-                new Filter<T>(x => x.ID).InList(ids),
-                DeletionData.DeletionColumns<T>()).Rows.Select(x => x.ToObject<T>()).ToList();
-            if (!entityList.Any())
-                return;
 
-            var deletionData = new DeletionData();
-            foreach (var entity in entityList)
+            if (typeof(T).HasAttribute<UnrecoverableAttribute>())
             {
-                deletionData.DeleteEntity(entity);
-            }
-            CascadeDelete(typeof(T), ids, deletionData);
+                Purge(entities);
 
-            var tableName = typeof(T).Name;
-            var deletion = new Deletion()
+                var props = GetExternalProperties(typeof(T));
+                foreach(var prop in props)
+                {
+                    foreach(var entity in entities)
+                    {
+                        DeleteExternalData(typeof(T), prop.Name, entity.ID);
+                    }
+                }
+            }
+            else
             {
-                DeletionDate = DateTime.Now,
-                HeadTable = tableName,
-                Description = $"Deleted {entityList.Count} entries",
-                DeletedBy = userID,
-                Data = Serialization.Serialize(deletionData)
-            };
-            OnSave(deletion);
-            Purge(entities);
+                var ids = entities.Select(x => x.ID).ToArray();
+                var entityList = Query(
+                    new Filter<T>(x => x.ID).InList(ids),
+                    DeletionData.DeletionColumns<T>()).Rows.Select(x => x.ToObject<T>()).ToList();
+                if (!entityList.Any())
+                    return;
+
+                var deletionData = new DeletionData();
+                foreach (var entity in entityList)
+                {
+                    deletionData.DeleteEntity(entity);
+                }
+                CascadeDelete(typeof(T), ids, deletionData);
+
+                var tableName = typeof(T).Name;
+                var deletion = new Deletion()
+                {
+                    DeletionDate = DateTime.Now,
+                    HeadTable = tableName,
+                    Description = $"Deleted {entityList.Count} entries",
+                    DeletedBy = userID,
+                    Data = Serialization.Serialize(deletionData)
+                };
+                OnSave(deletion);
+                Purge(entities);
+            }
         }
 
         private void AddDeletionType(Type type, List<Type> deletions)
@@ -2990,6 +3087,24 @@ namespace InABox.Database.SQLite
         
         public void Purge(Deletion deletion)
         {
+            var data = Serialization.Deserialize<DeletionData>(deletion.Data);
+            if(data is not null)
+            {
+                foreach(var (entityName, cascade) in data.Cascades)
+                {
+                    if (!CoreUtils.TryGetEntity(entityName, out var entityType)) continue;
+
+                    var props = GetExternalProperties(entityType);
+                    foreach(var prop in props)
+                    {
+                        foreach(var entity in cascade.ToObjects(entityType).Cast<Entity>())
+                        {
+                            DeleteExternalData(entityType, prop.Name, entity.ID);
+                        }
+                    }
+                }
+            }
+
             Purge<Deletion>(deletion);
         }
 
@@ -3055,5 +3170,52 @@ namespace InABox.Database.SQLite
 
         #endregion
 
+        #region External Data Storage
+
+        private string ExternalDataFolder(Type T, string columnName, string idString)
+        {
+            return Path.Combine(
+                Path.GetDirectoryName(URL) ?? "",
+                $"{Path.GetFileName(URL)}.data",
+                T.Name,
+                columnName,
+                idString.Substring(0, 2));
+        }
+
+        private byte[]? GetExternalData(Type T, string columnName, Guid id)
+        {
+            var idString = id.ToString();
+            var filename = Path.Combine(ExternalDataFolder(T, columnName, idString), idString);
+            try
+            {
+                return File.ReadAllBytes(filename);
+            }
+            catch(Exception e)
+            {
+                //Logger.Send(LogType.Error, "", $"Could not load external {T.Name}.{columnName}: {e.Message}");
+                return null;
+            }
+        }
+        private void SaveExternalData(Type T, string columnName, Guid id, byte[] data)
+        {
+            var idString = id.ToString();
+            var directory = ExternalDataFolder(T, columnName, idString);
+            Directory.CreateDirectory(directory);
+            var filename = Path.Combine(directory, idString);
+            File.WriteAllBytes(filename, data);
+        }
+        private void DeleteExternalData(Type T, string columnName, Guid id)
+        {
+            var idString = id.ToString();
+            var directory = ExternalDataFolder(T, columnName, idString);
+            Directory.CreateDirectory(directory);
+            var filename = Path.Combine(directory, idString);
+            if (File.Exists(filename))
+            {
+                File.Delete(filename);
+            }
+        }
+
+        #endregion
     }
 }

inabox.wpf/DynamicGrid/DynamicGridCustomColumnsComponent.cs

@@ -74,21 +74,6 @@ public class DynamicGridCustomColumnsComponent<T>
         new UserConfiguration<DynamicGridColumns>(tag).Save(columns);
     }
 
-    public void LoadColumnsMenu(ContextMenu menu)
-    {
-        menu.AddSeparatorIfNeeded();
-        var ResetColumns = new MenuItem { Header = "Reset Columns to Default" };
-        ResetColumns.Click += ResetColumnsClick;
-        menu.Items.Add(ResetColumns);
-        if (Security.IsAllowed<CanSetDefaultColumns>())
-        {
-            menu.Items.Add(new Separator());
-            var UpdateDefaultColumns = new MenuItem { Header = "Mark Columns as Default" };
-            UpdateDefaultColumns.Click += UpdateDefaultColumnsClick;
-            menu.Items.Add(UpdateDefaultColumns);
-        }
-    }
-
     private string GetTag(bool directEdit)
     {
         var tag = Tag ?? typeof(T).Name;
@@ -115,14 +100,32 @@ public class DynamicGridCustomColumnsComponent<T>
         return new NullEditor();
     }
 
-    private void ResetColumnsClick(object sender, RoutedEventArgs e)
+    public void LoadColumnsMenu(ContextMenu menu)
+    {
+        menu.AddSeparatorIfNeeded();
+        menu.AddItem("Reset Columns to Default", null, ResetColumnsClick);
+        if (Security.IsAllowed<CanSetDefaultColumns>())
+        {
+            menu.AddItem("Reset Columns to System Default", null, ResetColumnsToSystemClick);
+            menu.AddSeparator();
+            menu.AddItem("Mark Columns as Default", null, UpdateDefaultColumnsClick);
+        }
+    }
+
+    private void ResetColumnsToSystemClick()
+    {
+        SaveColumns(Grid.GenerateColumns());
+        Grid.Refresh(true, false);
+    }
+
+    private void ResetColumnsClick()
     {
         Grid.VisibleColumns.Clear();
         SaveColumns(Grid.VisibleColumns);
         Grid.Refresh(true, true);
     }
 
-    private void UpdateDefaultColumnsClick(object sender, RoutedEventArgs e)
+    private void UpdateDefaultColumnsClick()
     {
         var tag = GetTag(Grid.IsDirectEditMode());
         new GlobalConfiguration<DynamicGridColumns>(tag).Save(Grid.VisibleColumns);

inabox.wpf/DynamicGrid/DynamicManyToManyGrid.cs

@@ -19,7 +19,10 @@ public interface IDynamicManyToManyGrid<TManyToMany, TThis> : IDynamicEditorPage
 {
 }
 
-public class DynamicManyToManyGrid<TManyToMany, TThis> : DynamicGrid<TManyToMany>, IDynamicEditorPage, IDynamicManyToManyGrid<TManyToMany, TThis>
+public class DynamicManyToManyGrid<TManyToMany, TThis> : DynamicGrid<TManyToMany>,
+    IDynamicEditorPage,
+    IDynamicManyToManyGrid<TManyToMany, TThis>,
+    IDynamicMemoryEntityGrid<TManyToMany>
     where TThis : Entity, new()
     where TManyToMany : Entity, IPersistent, IRemotable, new()
 {
@@ -39,6 +42,8 @@ public class DynamicManyToManyGrid<TManyToMany, TThis> : DynamicGrid<TManyToMany
     
     protected List<TManyToMany> WorkingList = new();
 
+    IEnumerable<TManyToMany> IDynamicMemoryEntityGrid<TManyToMany>.Items => WorkingList;
+
     public PageType PageType => PageType.Other;
 
     private bool _readOnly;
@@ -59,16 +64,7 @@ public class DynamicManyToManyGrid<TManyToMany, TThis> : DynamicGrid<TManyToMany
 
     protected DynamicGridCustomColumnsComponent<TManyToMany> ColumnsComponent;
 
-    /// <summary>
-    /// A set of columns representing which columns have been loaded from the database.
-    /// </summary>
-    /// <remarks>
-    /// This is used to refresh the data when the columns change.<br/>
-    /// 
-    /// It is <see langword="null"/> if no data has been loaded from the database (that is, the data was gotten from
-    /// a page data handler instead.)
-    /// </remarks>
-    private HashSet<string>? LoadedColumns;
+    public HashSet<string>? LoadedColumns { get; set; }
 
     public DynamicManyToManyGrid()
     {
@@ -380,31 +376,7 @@ public class DynamicManyToManyGrid<TManyToMany, TThis> : DynamicGrid<TManyToMany
         var results = new CoreTable();
         results.LoadColumns(typeof(TManyToMany));
 
-        if (LoadedColumns is not null)
-        {
-            // Figure out which columns we still need.
-            var newColumns = columns.Where(x => !LoadedColumns.Contains(x.Property)).ToColumns();
-            if (newColumns.Any() && typeof(TManyToMany).GetCustomAttribute<AutoEntity>() is null)
-            {
-                var data = Client.Query(
-                    new Filter<TManyToMany>(x => x.ID).InList(WorkingList.Select(x => x.ID).Where(x => x != Guid.Empty).ToArray()),
-                    // We also need to add ID, so we know which item to fill.
-                    newColumns.Add(x => x.ID));
-                foreach (var row in data.Rows)
-                {
-                    var item = WorkingList.FirstOrDefault(x => x.ID == row.Get<TManyToMany, Guid>(y => y.ID));
-                    if (item is not null)
-                    {
-                        row.FillObject(item, overrideExisting: false);
-                    }
-                }
-                // Remember that we have now loaded this data.
-                foreach (var column in newColumns)
-                {
-                    LoadedColumns.Add(column.Property);
-                }
-            }
-        }
+        this.EnsureColumns(columns);
 
         if (sort != null)
         {

inabox.wpf/DynamicGrid/DynamicOneToManyGrid.cs

@@ -14,400 +14,372 @@ using InABox.Core;
 using InABox.Core.Reports;
 using InABox.WPF;
 
-namespace InABox.DynamicGrid
+namespace InABox.DynamicGrid;
+
+public interface IDynamicOneToManyGrid<TOne, TMany> : IDynamicEditorPage
 {
-    public interface IDynamicOneToManyGrid<TOne, TMany> : IDynamicEditorPage
-    {
-        List<TMany> Items { get; }
-        void LoadItems(TMany[] items);
-    }
+    List<TMany> Items { get; }
+    void LoadItems(TMany[] items);
+}
+
+public class DynamicOneToManyGrid<TOne, TMany> : DynamicGrid<TMany>,
+    IDynamicEditorPage,
+    IDynamicOneToManyGrid<TOne, TMany>,
+    IDynamicMemoryEntityGrid<TMany>
+    where TOne : Entity, new() where TMany : Entity, IPersistent, IRemotable, new()
+{
+    private TMany[] MasterList = Array.Empty<TMany>();
+    private readonly PropertyInfo property;
 
-    public class DynamicOneToManyGrid<TOne, TMany> : DynamicGrid<TMany>, IDynamicEditorPage, IDynamicOneToManyGrid<TOne, TMany>
-        where TOne : Entity, new() where TMany : Entity, IPersistent, IRemotable, new()
-    {
-        private TMany[] MasterList = Array.Empty<TMany>();
-        private readonly PropertyInfo property;
+    public HashSet<string>? LoadedColumns { get; set; }
 
-        /// <summary>
-        /// A set of columns representing which columns have been loaded from the database.
-        /// </summary>
-        /// <remarks>
-        /// This is used to refresh the data when the columns change.<br/>
-        /// 
-        /// It is <see langword="null"/> if no data has been loaded from the database (that is, the data was gotten from
-        /// a page data handler instead.)
-        /// </remarks>
-        private HashSet<string>? LoadedColumns;
+    IEnumerable<TMany> IDynamicMemoryEntityGrid<TMany>.Items => Items;
 
-        protected DynamicGridCustomColumnsComponent<TMany> ColumnsComponent;
+    protected DynamicGridCustomColumnsComponent<TMany> ColumnsComponent;
 
-        public DynamicOneToManyGrid()
-        {
-            Ready = false;
-            Items = new List<TMany>();
-            Criteria = new Filters<TMany>();
+    public DynamicOneToManyGrid()
+    {
+        Ready = false;
+        Items = new List<TMany>();
+        Criteria = new Filters<TMany>();
 
-            property = CoreUtils.GetOneToManyProperty(typeof(TMany), typeof(TOne));
+        property = CoreUtils.GetOneToManyProperty(typeof(TMany), typeof(TOne));
 
-            AddHiddenColumn(property.Name + "." + nameof(IEntityLink.ID));
-            foreach (var col in LookupFactory.RequiredColumns<TMany>())
-                HiddenColumns.Add(col);
+        AddHiddenColumn(property.Name + "." + nameof(IEntityLink.ID));
+        foreach (var col in LookupFactory.RequiredColumns<TMany>())
+            HiddenColumns.Add(col);
 
-            ColumnsComponent = new DynamicGridCustomColumnsComponent<TMany>(this, GetTag());
-        }
+        ColumnsComponent = new DynamicGridCustomColumnsComponent<TMany>(this, GetTag());
+    }
 
-        protected override void Init()
-        {
-        }
+    protected override void Init()
+    {
+    }
 
-        protected override void DoReconfigure(FluentList<DynamicGridOption> options)
-        {
-            options.BeginUpdate();
-
-            options.Add(DynamicGridOption.RecordCount)
-                .Add(DynamicGridOption.SelectColumns);
-
-            if (Security.CanEdit<TMany>() && !ReadOnly)
-                options.Add(DynamicGridOption.AddRows).Add(DynamicGridOption.EditRows);
-            if (Security.CanDelete<TMany>() && !ReadOnly)
-                options.Add(DynamicGridOption.DeleteRows);
-            if (Security.CanImport<TMany>() && !ReadOnly)
-                options.Add(DynamicGridOption.ImportData);
-            if (Security.CanExport<TMany>())
-                options.Add(DynamicGridOption.ExportData);
-            if (Security.CanMerge<TMany>())
-                options.Add(DynamicGridOption.MultiSelect);
-
-            options.EndUpdate();
-        }
+    protected override void DoReconfigure(FluentList<DynamicGridOption> options)
+    {
+        options.BeginUpdate();
+
+        options.Add(DynamicGridOption.RecordCount)
+            .Add(DynamicGridOption.SelectColumns);
+        
+        if (Security.CanEdit<TMany>() && !ReadOnly)
+            options.Add(DynamicGridOption.AddRows).Add(DynamicGridOption.EditRows);
+        if (Security.CanDelete<TMany>() && !ReadOnly)
+            options.Add(DynamicGridOption.DeleteRows);
+        if (Security.CanImport<TMany>() && !ReadOnly)
+            options.Add(DynamicGridOption.ImportData);
+        if (Security.CanExport<TMany>())
+            options.Add(DynamicGridOption.ExportData);
+        if (Security.CanMerge<TMany>())
+            options.Add(DynamicGridOption.MultiSelect);
+
+        options.EndUpdate();
+    }
 
-        private static bool IsAutoEntity => typeof(TMany).HasAttribute<AutoEntity>();
+    private static bool IsAutoEntity => typeof(TMany).HasAttribute<AutoEntity>();
 
-        protected Filters<TMany> Criteria { get; } = new Filters<TMany>();
+    protected Filters<TMany> Criteria { get; } = new Filters<TMany>();
 
-        public TOne Item { get; protected set; }
+    public TOne Item { get; protected set; }
 
-        public List<TMany> Items { get; private set; }
+    public List<TMany> Items { get; private set; }
 
-        public void LoadItems(TMany[] items)
-        {
-            Items.Clear();
-            Items.AddRange(items);
-            Refresh(false, true);
-        }
+    public void LoadItems(TMany[] items)
+    {
+        Items.Clear();
+        Items.AddRange(items);
+        Refresh(false, true);
+    }
 
-        private static string GetTag()
-        {
-            return typeof(TOne).Name + "." + typeof(TMany).Name;
-        }
+    private static string GetTag()
+    {
+        return typeof(TOne).Name + "." + typeof(TMany).Name;
+    }
 
-        #region IDynamicEditorPage
+    #region IDynamicEditorPage
 
-        public DynamicEditorGrid EditorGrid { get; set; }
+    public DynamicEditorGrid EditorGrid { get; set; }
 
-        public PageType PageType => PageType.Other;
+    public PageType PageType => PageType.Other;
 
-        public bool Ready { get; set; }
+    public bool Ready { get; set; }
 
-        private bool _readOnly;
-        public bool ReadOnly
+    private bool _readOnly;
+    public bool ReadOnly
+    {
+        get => _readOnly;
+        set
         {
-            get => _readOnly;
-            set
+            if (_readOnly != value)
             {
-                if (_readOnly != value)
-                {
-                    _readOnly = value;
-                    Reconfigure();
-                }
+                _readOnly = value;
+                Reconfigure();
             }
         }
+    }
 
-        public virtual void Load(object item, Func<Type, CoreTable?>? PageDataHandler)
-        {
-            Reconfigure();
+    public virtual void Load(object item, Func<Type, CoreTable?>? PageDataHandler)
+    {
+        Reconfigure();
 
-            Item = (TOne)item;
+        Item = (TOne)item;
 
-            Refresh(true, false);
+        Refresh(true, false);
 
-            var data = PageDataHandler?.Invoke(typeof(TMany));
+        var data = PageDataHandler?.Invoke(typeof(TMany));
 
-            if (data == null)
+        if (data == null)
+        {
+            if (Item.ID == Guid.Empty)
             {
-                if (Item.ID == Guid.Empty)
-                {
-                    data = new CoreTable();
-                    data.LoadColumns(typeof(TMany));
-                }
-                else
-                {
-                    var criteria = new Filters<TMany>();
-                    var exp = CoreUtils.GetPropertyExpression<TMany>(property.Name + ".ID");
-                    criteria.Add(new Filter<TMany>(exp).IsEqualTo(Item.ID).And(exp).IsNotEqualTo(Guid.Empty));
-                    criteria.AddRange(Criteria.Items);
-                    var sort = LookupFactory.DefineSort<TMany>();
-
-                    var columns = DynamicGridUtils.LoadEditorColumns(DataColumns());
-
-                    data = Client.Query(criteria.Combine(), columns, sort);
-
-                    LoadedColumns = columns.ColumnNames().ToHashSet();
-                }
+                data = new CoreTable();
+                data.LoadColumns(typeof(TMany));
             }
+            else
+            {
+                var criteria = new Filters<TMany>();
+                var exp = CoreUtils.GetPropertyExpression<TMany>(property.Name + ".ID");
+                criteria.Add(new Filter<TMany>(exp).IsEqualTo(Item.ID).And(exp).IsNotEqualTo(Guid.Empty));
+                criteria.AddRange(Criteria.Items);
+                var sort = LookupFactory.DefineSort<TMany>();
 
-            MasterList = data.Rows.Select(x => x.ToObject<TMany>()).ToArray();
+                var columns = DynamicGridUtils.LoadEditorColumns(DataColumns());
 
-            Items = MasterList.ToList();
-            Refresh(false, true);
-            Ready = true;
-        }
+                data = Client.Query(criteria.Combine(), columns, sort);
 
-        public virtual void BeforeSave(object item)
-        {
-            // Don't need to do anything here
+                LoadedColumns = columns.ColumnNames().ToHashSet();
+            }
         }
 
-        public virtual void AfterSave(object item)
-        {
-            if (IsAutoEntity)
-            {
-                return;
-            }
-            // First remove any deleted files
-            foreach (var map in MasterList)
-                if (!Items.Contains(map))
-                    OnDeleteItem(map);
+        MasterList = data.Rows.Select(x => x.ToObject<TMany>()).ToArray();
 
-            foreach (var map in Items)
-            {
-                var prop = (property.GetValue(map) as IEntityLink)!;
-                prop.ID = Item.ID;
-                prop.Synchronise(Item);
-            }
+        Items = MasterList.ToList();
+        Refresh(false, true);
+        Ready = true;
+    }
 
-            new Client<TMany>().Save(Items.Where(x => x.IsChanged()), "Updated by User");
-        }
+    public virtual void BeforeSave(object item)
+    {
+        // Don't need to do anything here
+    }
 
-        public Size MinimumSize()
+    public virtual void AfterSave(object item)
+    {
+        if (IsAutoEntity)
         {
-            return new Size(400, 400);
+            return;
         }
+        // First remove any deleted files
+        foreach (var map in MasterList)
+            if (!Items.Contains(map))
+                OnDeleteItem(map);
 
-        public string Caption()
+        foreach (var map in Items)
         {
-            var caption = typeof(TMany).GetCustomAttribute(typeof(Caption));
-            if (caption != null)
-                return ((Caption)caption).Text;
-            var result = new Inflector.Inflector(new CultureInfo("en")).Pluralize(typeof(TMany).Name);
-            return result;
+            var prop = (property.GetValue(map) as IEntityLink)!;
+            prop.ID = Item.ID;
+            prop.Synchronise(Item);
         }
 
-        public virtual int Order()
-        {
-            return int.MinValue;
-        }
+        new Client<TMany>().Save(Items.Where(x => x.IsChanged()), "Updated by User");
+    }
 
-        #endregion
+    public Size MinimumSize()
+    {
+        return new Size(400, 400);
+    }
 
-        #region DynamicGrid
+    public string Caption()
+    {
+        var caption = typeof(TMany).GetCustomAttribute(typeof(Caption));
+        if (caption != null)
+            return ((Caption)caption).Text;
+        var result = new Inflector.Inflector(new CultureInfo("en")).Pluralize(typeof(TMany).Name);
+        return result;
+    }
 
-        protected virtual void OnDeleteItem(TMany item)
-        {
-            if (IsAutoEntity)
-            {
-                return;
-            }
-            Client.Delete(item, typeof(TMany).Name + " Deleted by User");
-        }
+    public virtual int Order()
+    {
+        return int.MinValue;
+    }
 
+    #endregion
 
-        protected override CoreTable LoadImportKeys(string[] fields)
+    #region DynamicGrid
+
+    protected virtual void OnDeleteItem(TMany item)
+    {
+        if (IsAutoEntity)
         {
-            var result = base.LoadImportKeys(fields);
-            result.LoadRows(MasterList);
-            return result;
+            return;
         }
+        Client.Delete(item, typeof(TMany).Name + " Deleted by User");
+    }
 
-        protected override bool CustomiseImportItem(TMany item)
-        {
-            var result = base.CustomiseImportItem(item);
-            if (result)
-            {
-                var prop = (property.GetValue(item) as IEntityLink)!;
-                prop.ID = Item.ID;
-                prop.Synchronise(Item);
-            }
 
-            return result;
-        }
-        public override DynamicGridColumns GenerateColumns()
-        {
-            var cols = new DynamicGridColumns();
-            cols.AddRange(base.GenerateColumns().Where(x => !x.ColumnName.StartsWith(property.Name + ".")));
-            return cols;
-        }
+    protected override CoreTable LoadImportKeys(string[] fields)
+    {
+        var result = base.LoadImportKeys(fields);
+        result.LoadRows(MasterList);
+        return result;
+    }
 
-        protected override DynamicGridColumns LoadColumns()
+    protected override bool CustomiseImportItem(TMany item)
+    {
+        var result = base.CustomiseImportItem(item);
+        if (result)
         {
-            return ColumnsComponent.LoadColumns();
+            var prop = (property.GetValue(item) as IEntityLink)!;
+            prop.ID = Item.ID;
+            prop.Synchronise(Item);
         }
 
-        protected override void SaveColumns(DynamicGridColumns columns)
-        {
-            ColumnsComponent.SaveColumns(columns);
-        }
-        protected override void LoadColumnsMenu(ContextMenu menu)
-        {
-            base.LoadColumnsMenu(menu);
-            ColumnsComponent.LoadColumnsMenu(menu);
-        }
+        return result;
+    }
+    public override DynamicGridColumns GenerateColumns()
+    {
+        var cols = new DynamicGridColumns();
+        cols.AddRange(base.GenerateColumns().Where(x => !x.ColumnName.StartsWith(property.Name + ".")));
+        return cols;
+    }
 
-        protected override DynamicGridSettings LoadSettings()
-        {
-            var tag = GetTag();
+    protected override DynamicGridColumns LoadColumns()
+    {
+        return ColumnsComponent.LoadColumns();
+    }
 
-            var user = Task.Run(() => new UserConfiguration<DynamicGridSettings>(tag).Load());
-            user.Wait();
+    protected override void SaveColumns(DynamicGridColumns columns)
+    {
+        ColumnsComponent.SaveColumns(columns);
+    }
+    protected override void LoadColumnsMenu(ContextMenu menu)
+    {
+        base.LoadColumnsMenu(menu);
+        ColumnsComponent.LoadColumnsMenu(menu);
+    }
 
-            return user.Result;
-        }
-        protected override void SaveSettings(DynamicGridSettings settings)
-        {
-            var tag = GetTag();
-            new UserConfiguration<DynamicGridSettings>(tag).Save(settings);
-        }
+    protected override DynamicGridSettings LoadSettings()
+    {
+        var tag = GetTag();
 
-        protected override TMany CreateItem()
-        {
-            var result = new TMany();
-            var prop = (property.GetValue(result) as IEntityLink)!;
-            prop.ID = Item.ID;
-            prop.Synchronise(Item);
-            return result;
-        }
+        var user = Task.Run(() => new UserConfiguration<DynamicGridSettings>(tag).Load());
+        user.Wait();
 
-        protected override TMany LoadItem(CoreRow row)
-        {
-            return Items[_recordmap[row].Index];
-        }
+        return user.Result;
+    }
+    protected override void SaveSettings(DynamicGridSettings settings)
+    {
+        var tag = GetTag();
+        new UserConfiguration<DynamicGridSettings>(tag).Save(settings);
+    }
 
-        protected override TMany[] LoadItems(CoreRow[] rows)
-        {
-            var result = new List<TMany>();
-            foreach (var row in rows)
-                result.Add(LoadItem(row));
-            return result.ToArray();
-        }
+    protected override TMany CreateItem()
+    {
+        var result = new TMany();
+        var prop = (property.GetValue(result) as IEntityLink)!;
+        prop.ID = Item.ID;
+        prop.Synchronise(Item);
+        return result;
+    }
 
-        public override void SaveItem(TMany item)
-        {
-            if (!Items.Contains(item))
-                Items.Add(item);
+    protected override TMany LoadItem(CoreRow row)
+    {
+        return Items[_recordmap[row].Index];
+    }
 
-            if (item is ISequenceable) Items = Items.AsQueryable().OrderBy(x => (x as ISequenceable)!.Sequence).ToList();
-        }
+    protected override TMany[] LoadItems(CoreRow[] rows)
+    {
+        var result = new List<TMany>();
+        foreach (var row in rows)
+            result.Add(LoadItem(row));
+        return result.ToArray();
+    }
+
+    public override void SaveItem(TMany item)
+    {
+        if (!Items.Contains(item))
+            Items.Add(item);
+
+        if (item is ISequenceable) Items = Items.AsQueryable().OrderBy(x => (x as ISequenceable)!.Sequence).ToList();
+    }
 
-        protected override void DeleteItems(params CoreRow[] rows)
+    protected override void DeleteItems(params CoreRow[] rows)
+    {
+        var items = rows.Select(LoadItem).ToList();
+        foreach (var item in items)
         {
-            var items = rows.Select(LoadItem).ToList();
-            foreach (var item in items)
-            {
-                Items.Remove(item);
-            }
+            Items.Remove(item);
         }
+    }
 
-        protected override void Reload(Filters<TMany> criteria, Columns<TMany> columns, ref SortOrder<TMany>? sort,
-            Action<CoreTable?, Exception?> action)
-        {
-            var results = new CoreTable();
-            results.LoadColumns(typeof(TMany));
 
-            if (LoadedColumns is not null)
-            {
-                // Figure out which columns we still need.
-                var newColumns = columns.Where(x => !LoadedColumns.Contains(x.Property)).ToColumns();
-                if (newColumns.Any() && typeof(TMany).GetCustomAttribute<AutoEntity>() is null)
-                {
-                    var data = Client.Query(
-                        new Filter<TMany>(x => x.ID).InList(Items.Select(x => x.ID).Where(x => x != Guid.Empty).ToArray()),
-                        // We also need to add ID, so we know which item to fill.
-                        newColumns.Add(x => x.ID));
-                    foreach (var row in data.Rows)
-                    {
-                        var item = Items.FirstOrDefault(x => x.ID == row.Get<TMany, Guid>(y => y.ID));
-                        if (item is not null)
-                        {
-                            row.FillObject(item, overrideExisting: false);
-                        }
-                    }
-                    // Remember that we have now loaded this data.
-                    foreach (var column in newColumns)
-                    {
-                        LoadedColumns.Add(column.Property);
-                    }
-                }
-            }
+    protected override void Reload(Filters<TMany> criteria, Columns<TMany> columns, ref SortOrder<TMany>? sort,
+        Action<CoreTable?, Exception?> action)
+    {
+        var results = new CoreTable();
+        results.LoadColumns(typeof(TMany));
+
+        this.EnsureColumns(columns);
 
-            if (sort != null)
+        if (sort != null)
+        {
+            var exp = IQueryableExtensions.ToLambda<TMany>(sort.Expression);
+            var sorted = sort.Direction == SortDirection.Ascending
+                ? Items.AsQueryable().OrderBy(exp)
+                : Items.AsQueryable().OrderByDescending(exp);
+            foreach (var then in sort.Thens)
             {
-                var exp = IQueryableExtensions.ToLambda<TMany>(sort.Expression);
-                var sorted = sort.Direction == SortDirection.Ascending
-                    ? Items.AsQueryable().OrderBy(exp)
-                    : Items.AsQueryable().OrderByDescending(exp);
-                foreach (var then in sort.Thens)
-                {
-                    var thexp = IQueryableExtensions.ToLambda<TMany>(then.Expression);
-                    sorted = sort.Direction == SortDirection.Ascending ? sorted.ThenBy(exp) : sorted.ThenByDescending(exp);
-                }
-                Items = sorted.ToList();
+                var thexp = IQueryableExtensions.ToLambda<TMany>(then.Expression);
+                sorted = sort.Direction == SortDirection.Ascending ? sorted.ThenBy(exp) : sorted.ThenByDescending(exp);
             }
-            results.LoadRows(Items);
-
-            action.Invoke(results, null);
+            Items = sorted.ToList();
         }
+        results.LoadRows(Items);
 
-        protected override BaseEditor? GetEditor(object item, DynamicGridColumn column)
-        {
-            var type = CoreUtils.GetProperty(typeof(TMany), column.ColumnName).DeclaringType;
-            if (type.GetInterfaces().Contains(typeof(IEntityLink)) && type.ContainsInheritedGenericType(typeof(TOne)))
-                return new NullEditor();
-            return base.GetEditor(item, column);
-        }
+        action.Invoke(results, null);
+    }
 
-        public override void LoadEditorButtons(TMany item, DynamicEditorButtons buttons)
-        {
-            base.LoadEditorButtons(item, buttons);
-            if (ClientFactory.IsSupported<AuditTrail>())
-                buttons.Add("Audit Trail", Wpf.Resources.view.AsBitmapImage(), item, AuditTrailClick);
-        }
+    protected override BaseEditor? GetEditor(object item, DynamicGridColumn column)
+    {
+        var type = CoreUtils.GetProperty(typeof(TMany), column.ColumnName).DeclaringType;
+        if (type.GetInterfaces().Contains(typeof(IEntityLink)) && type.ContainsInheritedGenericType(typeof(TOne)))
+            return new NullEditor();
+        return base.GetEditor(item, column);
+    }
 
-        private void AuditTrailClick(object sender, object? item)
-        {
-            if (item is not TMany entity) return;
+    public override void LoadEditorButtons(TMany item, DynamicEditorButtons buttons)
+    {
+        base.LoadEditorButtons(item, buttons);
+        if (ClientFactory.IsSupported<AuditTrail>())
+            buttons.Add("Audit Trail", Wpf.Resources.view.AsBitmapImage(), item, AuditTrailClick);
+    }
 
-            var window = new AuditWindow(entity.ID);
-            window.ShowDialog();
-        }
+    private void AuditTrailClick(object sender, object? item)
+    {
+        if (item is not TMany entity) return;
 
-        public override DynamicEditorPages LoadEditorPages(TMany item)
-        {
-            return item.ID != Guid.Empty ? base.LoadEditorPages(item) : new DynamicEditorPages();
-        }
+        var window = new AuditWindow(entity.ID);
+        window.ShowDialog();
+    }
+
+    public override DynamicEditorPages LoadEditorPages(TMany item)
+    {
+        return item.ID != Guid.Empty ? base.LoadEditorPages(item) : new DynamicEditorPages();
+    }
 
-        protected override bool BeforePaste(IEnumerable<TMany> items, ClipAction action)
+    protected override bool BeforePaste(IEnumerable<TMany> items, ClipAction action)
+    {
+        if (action == ClipAction.Copy)
         {
-            if (action == ClipAction.Copy)
+            foreach (var item in items)
             {
-                foreach (var item in items)
-                {
-                    item.ID = Guid.Empty;
-                }
+                item.ID = Guid.Empty;
             }
-            return base.BeforePaste(items, action);
         }
+        return base.BeforePaste(items, action);
+    }
 
-        #endregion
+    #endregion
 
-    }
 }

inabox.wpf/DynamicGrid/IDynamicMemoryEntityGrid.cs

@@ -0,0 +1,170 @@
+using InABox.Clients;
+using InABox.Core;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace InABox.DynamicGrid;
+
+/// <summary>
+/// Defines a common interface for dealing with grids like <see cref="DynamicOneToManyGrid{TOne, TMany}"/>
+/// or <see cref="DynamicManyToManyGrid{TManyToMany, TThis}"/>, which display <see cref="Entity"/>s, but do not load them necessarily from the database,
+/// instead keeping them in memory.
+/// <br/>
+/// This interface then allows other functions, like
+/// <see cref="DynamicMemoryEntryGridExtensions.EnsureColumns{T}(InABox.DynamicGrid.IDynamicMemoryEntityGrid{T}, Columns{T})"/>, to work based on <see cref="IDynamicMemoryEntityGrid{T}.LoadedColumns"/> and manage our column handling better.
+/// </summary>
+/// <typeparam name="T"></typeparam>
+public interface IDynamicMemoryEntityGrid<T>
+    where T : Entity, IRemotable, IPersistent, new()
+{
+    /// <summary>
+    /// A set of columns representing which columns have been loaded from the database.
+    /// </summary>
+    /// <remarks>
+    /// This is used to refresh the data when the columns change.<br/>
+    /// 
+    /// It is <see langword="null"/> if no data has been loaded from the database (that is, the data was gotten from
+    /// a page data handler instead.)
+    /// </remarks>
+    public HashSet<string>? LoadedColumns { get; }
+
+    public IEnumerable<T> Items { get; }
+}
+
+public static class DynamicMemoryEntryGridExtensions
+{
+    public static void EnsureColumns<T>(this IDynamicMemoryEntityGrid<T> grid, Columns<T> columns)
+        where T : Entity, IRemotable, IPersistent, new()
+    {
+        RequireColumns(grid, columns);
+        LoadForeignProperties(grid, columns);
+    }
+
+    /// <summary>
+    /// Load the properties of any <see cref="EntityLink{T}"/>s on this <see cref="TMany"/> where the <see cref="IEntityLink.ID"/> is not <see cref="Guid.Empty"/>.
+    /// This allows us to populate columns of transient objects, as long as they are linked by the ID. What this actually then does is query each
+    /// linked table with the required columns.
+    /// </summary>
+    /// <param name="columns"></param>
+    public static void LoadForeignProperties<T>(this IDynamicMemoryEntityGrid<T> grid, Columns<T> columns)
+        where T : Entity, IRemotable, IPersistent, new()
+    {
+        // Lists of properties that we need, arranged by the entity link property which is their parent.
+        // LinkIDProperty : (Type, Properties: [(columnName, property)], Objects)
+        var newData = new Dictionary<IProperty, Tuple<Type, List<Tuple<string, IProperty>>, HashSet<T>>>();
+
+        foreach (var column in columns)
+        {
+            var property = DatabaseSchema.Property(typeof(T), column.Property);
+            if (property?.GetOuterParent(x => x.IsEntityLink) is IProperty linkProperty)
+            {
+                var remaining = column.Property[(linkProperty.Name.Length + 1)..];
+                if (remaining.Equals(nameof(IEntityLink.ID)))
+                {
+                    // This guy isn't foreign, so we don't pull him.
+                    continue;
+                }
+
+                var idProperty = DatabaseSchema.Property(typeof(T), linkProperty.Name + "." + nameof(IEntityLink.ID))!;
+
+                var linkType = linkProperty.PropertyType.GetInterfaceDefinition(typeof(IEntityLink<>))!.GenericTypeArguments[0];
+                if (!newData.TryGetValue(idProperty, out var data))
+                {
+                    data = new Tuple<Type, List<Tuple<string, IProperty>>, HashSet<T>>(
+                        linkType,
+                        new List<Tuple<string, IProperty>>(),
+                        new HashSet<T>());
+                    newData.Add(idProperty, data);
+                }
+
+                var any = false;
+                foreach (var item in grid.Items)
+                {
+                    if (!item.LoadedColumns.Contains(column.Property))
+                    {
+                        var linkID = (Guid)idProperty.Getter()(item);
+                        if (linkID != Guid.Empty)
+                        {
+                            any = true;
+                            data.Item3.Add(item);
+                        }
+                    }
+                }
+                if (any)
+                {
+                    data.Item2.Add(new(remaining, property));
+                }
+            }
+        }
+
+        var queryDefs = new List<IKeyedQueryDef>();
+        foreach (var (prop, data) in newData)
+        {
+            if (data.Item2.Any())
+            {
+                var ids = data.Item3.Select(prop.Getter()).Cast<Guid>().ToArray();
+                queryDefs.Add(new KeyedQueryDef(prop.Name, data.Item1,
+                    Filter.Create<Entity>(data.Item1, x => x.ID).InList(ids),
+                    Columns.Create(data.Item1, data.Item2.Select(x => x.Item1).ToArray()).Add<Entity>(x => x.ID)));
+            }
+        }
+        var results = Client.QueryMultiple(queryDefs);
+        foreach(var (prop, data) in newData)
+        {
+            var table = results.GetOrDefault(prop.Name);
+            if(table is null)
+            {
+                continue;
+            }
+            foreach (var entity in data.Item3)
+            {
+                var linkID = (Guid)prop.Getter()(entity);
+                var row = table.Rows.FirstOrDefault(x => x.Get<Entity, Guid>(x => x.ID) == linkID);
+                if (row is not null)
+                {
+                    foreach (var (name, property) in data.Item2)
+                    {
+                        if (!entity.LoadedColumns.Contains(property.Name))
+                        {
+                            property.Setter()(entity, row[name]);
+                            entity.LoadedColumns.Add(property.Name);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public static void RequireColumns<T>(this IDynamicMemoryEntityGrid<T> grid, Columns<T> columns)
+        where T : Entity, IRemotable, IPersistent, new()
+    {
+        if (grid.LoadedColumns is null) return;
+
+        // Figure out which columns we still need.
+        var newColumns = columns.Where(x => !grid.LoadedColumns.Contains(x.Property)).ToColumns();
+        if (newColumns.Any() && typeof(T).GetCustomAttribute<AutoEntity>() is null)
+        {
+            var data = Client.Query(
+                new Filter<T>(x => x.ID).InList(grid.Items.Select(x => x.ID).Where(x => x != Guid.Empty).ToArray()),
+                // We also need to add ID, so we know which item to fill.
+                newColumns.Add(x => x.ID));
+            foreach (var row in data.Rows)
+            {
+                var item = grid.Items.FirstOrDefault(x => x.ID == row.Get<T, Guid>(y => y.ID));
+                if (item is not null)
+                {
+                    row.FillObject(item, overrideExisting: false);
+                }
+            }
+            // Remember that we have now loaded this data.
+            foreach (var column in newColumns)
+            {
+                grid.LoadedColumns.Add(column.Property);
+            }
+        }
+    }
+}