DocumentCache system

InABox.Core/Classes/Document/IDocument.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace InABox.Core
+{
+    public interface IDocument : IEntity
+    {
+        public string FileName { get; set; }
+
+        public DateTime TimeStamp { get; set; }
+    }
+}

InABox.Core/DocumentCache.cs

@@ -0,0 +1,432 @@
+using InABox.Clients;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace InABox.Core
+{
+    public interface ICachedDocument
+    {
+        Guid ID { get; }
+
+        public void SerializeBinary(CoreBinaryWriter writer);
+
+        void DeserializeBinary(CoreBinaryReader reader, bool full);
+    }
+
+    public class CachedDocument<T> : ISerializeBinary
+        where T: ICachedDocument, new()
+    {
+        public Guid ID { get; set; }
+
+        public DateTime CachedAt { get; set; }
+
+        public T Document { get; set; }
+
+        private CachedDocument()
+        {
+        }
+
+        public CachedDocument(T document)
+        {
+            if (document.ID == Guid.Empty)
+            {
+                throw new Exception("Cannot cache document with no ID");
+            }
+            ID = document.ID;
+            CachedAt = DateTime.Now;
+            Document = document;
+        }
+
+        public void SerializeBinary(CoreBinaryWriter writer)
+        {
+            writer.Write(ID);
+            writer.Write(CachedAt);
+            Document.SerializeBinary(writer);
+        }
+
+        private void DeserializeBinary(CoreBinaryReader reader, bool full)
+        {
+            ID = reader.ReadGuid();
+            CachedAt = reader.ReadDateTime();
+            Document = new T();
+            Document.DeserializeBinary(reader, full);
+        }
+
+        public void DeserializeBinary(CoreBinaryReader reader)
+        {
+            DeserializeBinary(reader, true);
+        }
+
+        public static CachedDocument<T> ReadHeader(Stream stream)
+        {
+            var cache = new CachedDocument<T>();
+            cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: false);
+            return cache;
+        }
+
+        public static CachedDocument<T> ReadFull(Stream stream)
+        {
+            var cache = new CachedDocument<T>();
+            cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: true);
+            return cache;
+        }
+    }
+
+    public interface IDocumentCache
+    {
+        void Clear();
+
+        void ClearOld();
+    }
+
+    /// <summary>
+    /// A cache of documents that is saved in the AppData folder under a specific tag.
+    /// </summary>
+    /// <remarks>
+    /// The files are stored with the name "&lt;ID&gt;.document", and they contain a binary serialised <see cref="CachedDocument"/>.
+    /// This stores the date when the document was cached, allowing us to clear out old documents.
+    /// </remarks>
+    public abstract class DocumentCache<T> : IDocumentCache
+        where T : class, ICachedDocument, new()
+    {
+        public string Tag { get; set; }
+
+        public ConcurrentDictionary<Guid, byte> CachedDocuments { get; set; } = new ConcurrentDictionary<Guid, byte>();
+
+        public ConcurrentDictionary<Guid, byte> UncachedDocuments { get; set; } = new ConcurrentDictionary<Guid, byte>();
+
+        private bool _processing = false;
+        private object _processingLock = new object();
+
+        /// <summary>
+        /// How long before documents are allowed to be cleaned.
+        /// </summary>
+        public abstract TimeSpan MaxAge { get; }
+
+        public DocumentCache(string tag)
+        {
+            Tag = tag;
+
+            EnsureCacheFolder();
+            LoadCurrentCache();
+        }
+
+        #region Abstract Interface
+
+        protected abstract T? LoadDocument(Guid id);
+
+        #endregion
+
+        #region Public Interface
+
+        /// <summary>
+        /// Check if the cache contains a document with the given <paramref name="id"/>.
+        /// </summary>
+        /// <param name="id"></param>
+        /// <returns></returns>
+        public bool Has(Guid id)
+        {
+            return CachedDocuments.ContainsKey(id);
+        }
+
+        /// <summary>
+        /// Fetch a document from the cache or, if it hasn't been cached, through <see cref="LoadDocument(Guid)"/>.
+        /// </summary>
+        /// <param name="id"></param>
+        /// <returns><see langword="null"/> if <see cref="LoadDocument(Guid)"/> returned <see langword="null"/>.</returns>
+        public T? GetDocument(Guid id)
+        {
+            var file = GetFileName(id);
+            if (File.Exists(file))
+            {
+                using var stream = File.OpenRead(file);
+                var doc = CachedDocument<T>.ReadFull(stream);
+                return doc.Document;
+            }
+            var document = LoadDocument(id);
+            if(document != null)
+            {
+                Add(document);
+            }
+            return document;
+        }
+
+        /// <summary>
+        /// Add a loaded document to the cache.
+        /// </summary>
+        /// <param name="document"></param>
+        public void Add(T document)
+        {
+            var cached = new CachedDocument<T>(document);
+            using (var file = File.Open(GetFileName(cached.ID), FileMode.Create))
+            {
+                cached.WriteBinary(file, BinarySerializationSettings.Latest);
+            }
+            CachedDocuments.TryAdd(cached.ID, 0);
+        }
+
+        /// <summary>
+        /// Remove a document from the cache.
+        /// </summary>
+        /// <param name="id"></param>
+        public void Remove(Guid id)
+        {
+            File.Delete(GetFileName(id));
+            UncachedDocuments.TryRemove(id, out var _);
+            CachedDocuments.TryRemove(id, out _);
+        }
+
+        /// <summary>
+        /// Ensure that the cache contains <paramref name="documentIDs"/>. If it does not, a background worker will begin to download them.
+        /// </summary>
+        /// <param name="documentIDs"></param>
+        public void Ensure(IEnumerable<Guid> documentIDs)
+        {
+            foreach(var docID in documentIDs)
+            {
+                if (docID != Guid.Empty)
+                {
+                    if(!CachedDocuments.ContainsKey(docID))
+                    {
+                        UncachedDocuments.TryAdd(docID, 0);
+                    }
+                }
+            }
+            CheckProcessing();
+        }
+
+        /// <summary>
+        /// Like <see cref="Ensure(IEnumerable{Guid})"/>, but will clear out items that are not in <paramref name="documentIDs"/>.
+        /// </summary>
+        /// <param name="documentIDs"></param>
+        public void EnsureStrict(IList<Guid> documentIDs)
+        {
+            foreach (var docID in documentIDs)
+            {
+                if (docID != Guid.Empty)
+                {
+                    if (!CachedDocuments.ContainsKey(docID))
+                    {
+                        UncachedDocuments.TryAdd(docID, 0);
+                    }
+                }
+            }
+            ClearWhere(x => !documentIDs.Contains(x));
+            CheckProcessing();
+        }
+
+        /// <summary>
+        /// Clear all old cached documents, according to <see cref="MaxAge"/>.
+        /// </summary>
+        public void ClearOld()
+        {
+            ClearWhere(docID =>
+            {
+                var filename = GetFileName(docID);
+                if (File.Exists(filename))
+                {
+                    using var stream = File.OpenRead(filename);
+                    var doc = CachedDocument<T>.ReadHeader(stream);
+                    return DateTime.Now - doc.CachedAt > MaxAge;
+                }
+                else
+                {
+                    return true;
+                }
+            });
+        }
+
+        /// <summary>
+        /// Clear the entire cache.
+        /// </summary>
+        public void Clear()
+        {
+            foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document"))
+            {
+                File.Delete(file);
+            }
+            CachedDocuments.Clear();
+            UncachedDocuments.Clear();
+        }
+
+        #endregion
+
+        #region Private Methods
+
+        private void ClearWhere(Func<Guid, bool> predicate)
+        {
+            var toRemove = new List<Guid>();
+            foreach (var docID in CachedDocuments.Keys)
+            {
+                if (predicate(docID))
+                {
+                    File.Delete(GetFileName(docID));
+                    toRemove.Add(docID);
+                }
+            }
+            foreach (var id in toRemove)
+            {
+                CachedDocuments.TryRemove(id, out var _);
+            }
+        }
+
+        protected CachedDocument<T> GetHeader(Guid id)
+        {
+            var fileName = GetFileName(id);
+            using var stream = File.OpenRead(fileName);
+            return CachedDocument<T>.ReadHeader(stream);
+        }
+        protected CachedDocument<T> GetFull(Guid id)
+        {
+            var fileName = GetFileName(id);
+            using var stream = File.OpenRead(fileName);
+            return CachedDocument<T>.ReadFull(stream);
+        }
+
+        private void Process()
+        {
+            try
+            {
+                _processing = true;
+                while (true)
+                {
+                    Guid docID;
+                    lock (_processingLock)
+                    {
+                        docID = UncachedDocuments.Keys.FirstOrDefault();
+                        if (docID == Guid.Empty)
+                        {
+                            _processing = false;
+                            break;
+                        }
+                    }
+
+                    var document = LoadDocument(docID);
+                    if (document is null)
+                    {
+                        Logger.Send(LogType.Error, "", $"Document {docID} cannot be cached since it does not exist.");
+                    }
+                    else
+                    {
+                        Add(document);
+                    }
+
+                    UncachedDocuments.TryRemove(docID, out var _);
+                }
+            }
+            catch(Exception ex)
+            {
+                CoreUtils.LogException("", ex);
+                _processing = false;
+            }
+        }
+        
+        private void CheckProcessing()
+        {
+            lock (_processingLock)
+            {
+                if (!_processing && UncachedDocuments.Any())
+                {
+                    Task.Run(Process);
+                }
+            }
+        }
+
+        private void EnsureCacheFolder()
+        {
+            Directory.CreateDirectory(GetFolder());
+        }
+
+        private void LoadCurrentCache()
+        {
+            foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document"))
+            {
+                try
+                {
+                    using var stream = File.OpenRead(file);
+                    var doc = CachedDocument<T>.ReadHeader(stream);
+                    CachedDocuments.TryAdd(doc.ID, 0);
+                }
+                catch(Exception e)
+                {
+                    CoreUtils.LogException("", e, "Error loading cache");
+                    // Skip;
+                }
+            }
+        }
+
+        private string GetFolder()
+        {
+            return Path.Combine(CoreUtils.GetPath(), DocumentCaches.ServerName ?? "", "_documentcache", Tag);
+        }
+
+        private string GetFileName(Guid documentID)
+        {
+            return Path.Combine(GetFolder(), $"{documentID}.document");
+        }
+
+        #endregion
+    }
+
+    public static class DocumentCaches
+    {
+        private static readonly Dictionary<Type, IDocumentCache> Caches = new Dictionary<Type, IDocumentCache>();
+
+        public static string? ServerName { get; set; }
+
+        #region Registry
+
+        public static void RegisterAll()
+        {
+            var types = CoreUtils.TypeList(x => !x.IsAbstract && !x.IsGenericType && x.IsSubclassOf(typeof(IDocumentCache)));
+            foreach(var type in types)
+            {
+                Caches.Add(type, (Activator.CreateInstance(type) as IDocumentCache)!);
+            }
+        }
+
+        public static void RegisterCache<T>()
+            where T : IDocumentCache, new()
+        {
+            Caches.Add(typeof(T), new T());
+        }
+        public static T GetOrRegister<T>()
+            where T : class, IDocumentCache, new()
+        {
+            if(!Caches.TryGetValue(typeof(T), out var cache))
+            {
+                cache = new T();
+                Caches.Add(typeof(T), new T());
+            }
+            return (cache as T)!;
+        }
+
+        #endregion
+
+        #region Interface
+
+        public static void Clear()
+        {
+            foreach(var cache in Caches.Values)
+            {
+                cache.Clear();
+            }
+        }
+
+        public static void ClearOld()
+        {
+            foreach (var cache in Caches.Values)
+            {
+                cache.ClearOld();
+            }
+        }
+
+        #endregion
+    }
+}