inabox/InABox.Core/DocumentCache.cs

using InABox.Clients;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;

namespace InABox.Core
{
    public interface ICachedDocument
    {
        Guid ID { get; }

        public void SerializeBinary(CoreBinaryWriter writer);

        void DeserializeBinary(CoreBinaryReader reader, bool full);
    }

    public class CachedDocument<T> : ISerializeBinary
        where T: ICachedDocument, new()
    {
        public Guid ID { get; set; }

        public DateTime CachedAt { get; set; }

        public T Document { get; set; }

        private CachedDocument()
        {
        }

        public CachedDocument(T document)
        {
            if (document.ID == Guid.Empty)
            {
                throw new Exception("Cannot cache document with no ID");
            }
            ID = document.ID;
            CachedAt = DateTime.Now;
            Document = document;
        }

        public void SerializeBinary(CoreBinaryWriter writer)
        {
            writer.Write(ID);
            writer.Write(CachedAt);
            Document.SerializeBinary(writer);
        }

        private void DeserializeBinary(CoreBinaryReader reader, bool full)
        {
            ID = reader.ReadGuid();
            CachedAt = reader.ReadDateTime();
            Document = new T();
            Document.DeserializeBinary(reader, full);
        }

        public void DeserializeBinary(CoreBinaryReader reader)
        {
            DeserializeBinary(reader, true);
        }

        public static CachedDocument<T> ReadHeader(Stream stream)
        {
            var cache = new CachedDocument<T>();
            cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: false);
            return cache;
        }

        public static CachedDocument<T> ReadFull(Stream stream)
        {
            var cache = new CachedDocument<T>();
            cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: true);
            return cache;
        }
    }

    public interface IDocumentCache
    {
        void Clear();

        void ClearOld();
    }

    /// <summary>
    /// A cache of documents that is saved in the AppData folder under a specific tag. Contrary to the name, this isn't just a <see cref="Document"/> cache, but
    /// in fact a way to cache any kind of object that implements <see cref="ICachedDocument"/>.
    /// </summary>
    /// <remarks>
    /// The files are stored with the name "&lt;ID&gt;.document", and they contain a binary serialised <see cref="CachedDocument"/>.
    /// This stores the date when the document was cached, allowing us to clear out old documents.
    /// </remarks>
    public abstract class DocumentCache<T> : IDocumentCache
        where T : class, ICachedDocument, new()
    {
        public string Tag { get; set; }

        public ConcurrentDictionary<Guid, byte> CachedDocuments { get; set; } = new ConcurrentDictionary<Guid, byte>();

        public ConcurrentDictionary<Guid, byte> UncachedDocuments { get; set; } = new ConcurrentDictionary<Guid, byte>();

        private bool _processing = false;
        private object _processingLock = new object();

        /// <summary>
        /// How long before documents are allowed to be cleaned.
        /// </summary>
        public abstract TimeSpan MaxAge { get; }

        public DocumentCache(string tag)
        {
            Tag = tag;

            EnsureCacheFolder();
            LoadCurrentCache();
        }

        #region Abstract Interface

        protected abstract T? LoadDocument(Guid id);

        #endregion

        #region Public Interface

        /// <summary>
        /// Check if the cache contains a document with the given <paramref name="id"/>.
        /// </summary>
        /// <param name="id"></param>
        /// <returns></returns>
        public bool Has(Guid id)
        {
            return CachedDocuments.ContainsKey(id);
        }

        /// <summary>
        /// Fetch a document from the cache or, if it hasn't been cached, through <see cref="LoadDocument(Guid)"/>.
        /// </summary>
        /// <param name="id"></param>
        /// <returns><see langword="null"/> if <see cref="LoadDocument(Guid)"/> returned <see langword="null"/>.</returns>
        public T? GetDocument(Guid id)
        {
            var file = GetFileName(id);
            if (File.Exists(file))
            {
                using var stream = File.OpenRead(file);
                var doc = CachedDocument<T>.ReadFull(stream);
                return doc.Document;
            }
            var document = LoadDocument(id);
            if(document != null)
            {
                Add(document);
            }
            return document;
        }

        /// <summary>
        /// Add a loaded document to the cache.
        /// </summary>
        /// <param name="document"></param>
        public void Add(T document)
        {
            var cached = new CachedDocument<T>(document);
            using (var file = File.Open(GetFileName(cached.ID), FileMode.Create))
            {
                cached.WriteBinary(file, BinarySerializationSettings.Latest);
            }
            CachedDocuments.TryAdd(cached.ID, 0);
        }

        /// <summary>
        /// Remove a document from the cache.
        /// </summary>
        /// <param name="id"></param>
        public void Remove(Guid id)
        {
            File.Delete(GetFileName(id));
            UncachedDocuments.TryRemove(id, out var _);
            CachedDocuments.TryRemove(id, out _);
        }

        /// <summary>
        /// Ensure that the cache contains <paramref name="documentIDs"/>. If it does not, a background worker will begin to download them.
        /// </summary>
        /// <param name="documentIDs"></param>
        public void Ensure(IEnumerable<Guid> documentIDs)
        {
            foreach(var docID in documentIDs)
            {
                if (docID != Guid.Empty)
                {
                    if(!CachedDocuments.ContainsKey(docID))
                    {
                        UncachedDocuments.TryAdd(docID, 0);
                    }
                }
            }
            CheckProcessing();
        }

        /// <summary>
        /// Like <see cref="Ensure(IEnumerable{Guid})"/>, but will clear out items that are not in <paramref name="documentIDs"/>.
        /// </summary>
        /// <param name="documentIDs"></param>
        public void EnsureStrict(IList<Guid> documentIDs)
        {
            foreach (var docID in documentIDs)
            {
                if (docID != Guid.Empty)
                {
                    if (!CachedDocuments.ContainsKey(docID))
                    {
                        UncachedDocuments.TryAdd(docID, 0);
                    }
                }
            }
            ClearWhere(x => !documentIDs.Contains(x));
            CheckProcessing();
        }

        /// <summary>
        /// Clear all old cached documents, according to <see cref="MaxAge"/>.
        /// </summary>
        public void ClearOld()
        {
            ClearWhere(docID =>
            {
                var filename = GetFileName(docID);
                if (File.Exists(filename))
                {
                    using var stream = File.OpenRead(filename);
                    var doc = CachedDocument<T>.ReadHeader(stream);
                    return DateTime.Now - doc.CachedAt > MaxAge;
                }
                else
                {
                    return true;
                }
            });
        }

        /// <summary>
        /// Clear the entire cache.
        /// </summary>
        public void Clear()
        {
            foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document"))
            {
                File.Delete(file);
            }
            CachedDocuments.Clear();
            UncachedDocuments.Clear();
        }

        #endregion

        #region Private Methods

        private void ClearWhere(Func<Guid, bool> predicate)
        {
            var toRemove = new List<Guid>();
            foreach (var docID in CachedDocuments.Keys)
            {
                if (predicate(docID))
                {
                    File.Delete(GetFileName(docID));
                    toRemove.Add(docID);
                }
            }
            foreach (var id in toRemove)
            {
                CachedDocuments.TryRemove(id, out var _);
            }
        }

        protected CachedDocument<T> GetHeader(Guid id)
        {
            var fileName = GetFileName(id);
            using var stream = File.OpenRead(fileName);
            return CachedDocument<T>.ReadHeader(stream);
        }
        protected CachedDocument<T> GetFull(Guid id)
        {
            var fileName = GetFileName(id);
            using var stream = File.OpenRead(fileName);
            return CachedDocument<T>.ReadFull(stream);
        }

        private void Process()
        {
            try
            {
                _processing = true;
                while (true)
                {
                    Guid docID;
                    lock (_processingLock)
                    {
                        docID = UncachedDocuments.Keys.FirstOrDefault();
                        if (docID == Guid.Empty)
                        {
                            _processing = false;
                            break;
                        }
                    }

                    var document = LoadDocument(docID);
                    if (document is null)
                    {
                        Logger.Send(LogType.Error, "", $"Document {docID} cannot be cached since it does not exist.");
                    }
                    else
                    {
                        Add(document);
                    }

                    UncachedDocuments.TryRemove(docID, out var _);
                }
            }
            catch(Exception ex)
            {
                CoreUtils.LogException("", ex);
                _processing = false;
            }
        }
        
        private void CheckProcessing()
        {
            lock (_processingLock)
            {
                if (!_processing && UncachedDocuments.Any())
                {
                    Task.Run(Process);
                }
            }
        }

        private void EnsureCacheFolder()
        {
            Directory.CreateDirectory(GetFolder());
        }

        private void LoadCurrentCache()
        {
            foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document"))
            {
                try
                {
                    using var stream = File.OpenRead(file);
                    var doc = CachedDocument<T>.ReadHeader(stream);
                    CachedDocuments.TryAdd(doc.ID, 0);
                }
                catch(Exception e)
                {
                    CoreUtils.LogException("", e, "Error loading cache");
                    // Skip;
                }
            }
        }

        private string GetFolder()
        {
            return Path.Combine(
                CoreUtils.GetPath(),
                ClientFactory.DatabaseID.ToString(),
                "_documentcache",
                Tag);
        }

        private string GetFileName(Guid documentID)
        {
            return Path.Combine(GetFolder(), $"{documentID}.document");
        }

        #endregion
    }

    public static class DocumentCaches
    {
        private static readonly Dictionary<Type, IDocumentCache> Caches = new Dictionary<Type, IDocumentCache>();

        #region Registry

        public static void RegisterAll()
        {
            var types = CoreUtils.TypeList(x => !x.IsAbstract && !x.IsGenericType && x.IsSubclassOf(typeof(IDocumentCache)));
            foreach(var type in types)
            {
                Caches.Add(type, (Activator.CreateInstance(type) as IDocumentCache)!);
            }
        }

        public static void RegisterCache<T>()
            where T : IDocumentCache, new()
        {
            Caches.Add(typeof(T), new T());
        }
        public static T GetOrRegister<T>()
            where T : class, IDocumentCache, new()
        {
            if(!Caches.TryGetValue(typeof(T), out var cache))
            {
                cache = new T();
                Caches.Add(typeof(T), new T());
            }
            return (cache as T)!;
        }

        #endregion

        #region Interface

        public static void Clear()
        {
            foreach(var cache in Caches.Values)
            {
                cache.Clear();
            }
        }

        public static void ClearOld()
        {
            foreach (var cache in Caches.Values)
            {
                cache.ClearOld();
            }
        }

        #endregion
    }

    /// <summary>
    /// An implementation of <see cref="ICachedDocument"/> for use with entities of type <see cref="Document"/>. The <see cref="Document.TimeStamp"/>
    /// is saved along with the document, allowing us to refresh updated documents.
    /// </summary>
    public class DocumentCachedDocument : ICachedDocument
    {
        public DateTime TimeStamp { get; set; }

        public Document? Document { get; set; }

        public Guid ID => Document?.ID ?? Guid.Empty;

        public DocumentCachedDocument() { }

        public DocumentCachedDocument(Document document)
        {
            Document = document;
            TimeStamp = document.TimeStamp;
        }

        public void DeserializeBinary(CoreBinaryReader reader, bool full)
        {
            TimeStamp = reader.ReadDateTime();
            if (full)
            {
                Document = reader.ReadObject<Document>();
            }
        }

        public void SerializeBinary(CoreBinaryWriter writer)
        {
            writer.Write(TimeStamp);
            if (Document is null)
            {
                throw new Exception("Cannot serialize incomplete CachedDocument");
            }
            writer.WriteObject(Document);
        }
    }

    /// <summary>
    /// Implements a <see cref="DocumentCache{T}"/> for use with <see cref="Document"/>.
    /// </summary>
    public abstract class DocumentCache : DocumentCache<DocumentCachedDocument>
    {
        public DocumentCache(string tag): base(tag) { }

        protected override DocumentCachedDocument? LoadDocument(Guid id)
        {
            var document = Client.Query(new Filter<Document>(x => x.ID).IsEqualTo(id))
                .ToObjects<Document>().FirstOrDefault();
            if(document != null)
            {
                return new DocumentCachedDocument(document);
            }
            else
            {
                return null;
            }
        }
        
        /// <summary>
        /// Fetch a bunch of documents from the cache or the database, optionally checking against the timestamp listed in the database.
        /// </summary>
        /// <param name="ids"></param>
        /// <param name="checkTimestamp">
        /// If <see langword="true"/>, then loads <see cref="Document.TimeStamp"/> from the database for all cached documents,
        /// and if they are older, updates the cache.
        /// </param>
        public IEnumerable<Document> LoadDocuments(IEnumerable<Guid> ids, bool checkTimestamp = false)
        {
            var cached = new List<Guid>();
            var toLoad = new List<Guid>();
            foreach (var docID in ids)
            {
                if (Has(docID))
                {
                    cached.Add(docID);
                }
                else
                {
                    toLoad.Add(docID);
                }
            }

            var loadedCached = new List<Document>();
            if (cached.Count > 0)
            {
                var docs = Client.Query(
                    new Filter<Document>(x => x.ID).InList(cached.ToArray()),
                    Columns.None<Document>().Add(x => x.TimeStamp, x => x.ID));
                foreach (var doc in docs.ToObjects<Document>())
                {
                    try
                    {
                        var timestamp = GetHeader(doc.ID).Document.TimeStamp;
                        if (doc.TimeStamp > timestamp)
                        {
                            toLoad.Add(doc.ID);
                        }
                        else
                        {
                            loadedCached.Add(GetFull(doc.ID).Document.Document!);
                        }
                    }
                    catch (Exception e)
                    {
                        CoreUtils.LogException("", e, "Error loading cached file");
                        toLoad.Add(doc.ID);
                    }
                }
            }

            if (toLoad.Count > 0)
            {
                var loaded = Client.Query(new Filter<Document>(x => x.ID).InList(toLoad.ToArray()))
                    .ToObjects<Document>().ToList();
                foreach (var loadedDoc in loaded)
                {
                    Add(new DocumentCachedDocument(loadedDoc));
                }
                return loaded.Concat(loadedCached);
            }
            else
            {
                return loadedCached;
            }
        }
    }
}