using InABox.Clients; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; namespace InABox.Core { public interface ICachedDocument { Guid ID { get; } public void SerializeBinary(CoreBinaryWriter writer); void DeserializeBinary(CoreBinaryReader reader, bool full); } public class CachedDocument : ISerializeBinary where T: ICachedDocument, new() { public Guid ID { get; set; } public DateTime CachedAt { get; set; } public T Document { get; set; } private CachedDocument() { } public CachedDocument(T document) { if (document.ID == Guid.Empty) { throw new Exception("Cannot cache document with no ID"); } ID = document.ID; CachedAt = DateTime.Now; Document = document; } public void SerializeBinary(CoreBinaryWriter writer) { writer.Write(ID); writer.Write(CachedAt); Document.SerializeBinary(writer); } private void DeserializeBinary(CoreBinaryReader reader, bool full) { ID = reader.ReadGuid(); CachedAt = reader.ReadDateTime(); Document = new T(); Document.DeserializeBinary(reader, full); } public void DeserializeBinary(CoreBinaryReader reader) { DeserializeBinary(reader, true); } public static CachedDocument ReadHeader(Stream stream) { var cache = new CachedDocument(); cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: false); return cache; } public static CachedDocument ReadFull(Stream stream) { var cache = new CachedDocument(); cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: true); return cache; } } public interface IDocumentCache { void Clear(); void ClearOld(); } /// /// A cache of documents that is saved in the AppData folder under a specific tag. /// /// /// The files are stored with the name "<ID>.document", and they contain a binary serialised . /// This stores the date when the document was cached, allowing us to clear out old documents. /// public abstract class DocumentCache : IDocumentCache where T : class, ICachedDocument, new() { public string Tag { get; set; } public ConcurrentDictionary CachedDocuments { get; set; } = new ConcurrentDictionary(); public ConcurrentDictionary UncachedDocuments { get; set; } = new ConcurrentDictionary(); private bool _processing = false; private object _processingLock = new object(); /// /// How long before documents are allowed to be cleaned. /// public abstract TimeSpan MaxAge { get; } public DocumentCache(string tag) { Tag = tag; EnsureCacheFolder(); LoadCurrentCache(); } #region Abstract Interface protected abstract T? LoadDocument(Guid id); #endregion #region Public Interface /// /// Check if the cache contains a document with the given . /// /// /// public bool Has(Guid id) { return CachedDocuments.ContainsKey(id); } /// /// Fetch a document from the cache or, if it hasn't been cached, through . /// /// /// if returned . public T? GetDocument(Guid id) { var file = GetFileName(id); if (File.Exists(file)) { using var stream = File.OpenRead(file); var doc = CachedDocument.ReadFull(stream); return doc.Document; } var document = LoadDocument(id); if(document != null) { Add(document); } return document; } /// /// Add a loaded document to the cache. /// /// public void Add(T document) { var cached = new CachedDocument(document); using (var file = File.Open(GetFileName(cached.ID), FileMode.Create)) { cached.WriteBinary(file, BinarySerializationSettings.Latest); } CachedDocuments.TryAdd(cached.ID, 0); } /// /// Remove a document from the cache. /// /// public void Remove(Guid id) { File.Delete(GetFileName(id)); UncachedDocuments.TryRemove(id, out var _); CachedDocuments.TryRemove(id, out _); } /// /// Ensure that the cache contains . If it does not, a background worker will begin to download them. /// /// public void Ensure(IEnumerable documentIDs) { foreach(var docID in documentIDs) { if (docID != Guid.Empty) { if(!CachedDocuments.ContainsKey(docID)) { UncachedDocuments.TryAdd(docID, 0); } } } CheckProcessing(); } /// /// Like , but will clear out items that are not in . /// /// public void EnsureStrict(IList documentIDs) { foreach (var docID in documentIDs) { if (docID != Guid.Empty) { if (!CachedDocuments.ContainsKey(docID)) { UncachedDocuments.TryAdd(docID, 0); } } } ClearWhere(x => !documentIDs.Contains(x)); CheckProcessing(); } /// /// Clear all old cached documents, according to . /// public void ClearOld() { ClearWhere(docID => { var filename = GetFileName(docID); if (File.Exists(filename)) { using var stream = File.OpenRead(filename); var doc = CachedDocument.ReadHeader(stream); return DateTime.Now - doc.CachedAt > MaxAge; } else { return true; } }); } /// /// Clear the entire cache. /// public void Clear() { foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document")) { File.Delete(file); } CachedDocuments.Clear(); UncachedDocuments.Clear(); } #endregion #region Private Methods private void ClearWhere(Func predicate) { var toRemove = new List(); foreach (var docID in CachedDocuments.Keys) { if (predicate(docID)) { File.Delete(GetFileName(docID)); toRemove.Add(docID); } } foreach (var id in toRemove) { CachedDocuments.TryRemove(id, out var _); } } protected CachedDocument GetHeader(Guid id) { var fileName = GetFileName(id); using var stream = File.OpenRead(fileName); return CachedDocument.ReadHeader(stream); } protected CachedDocument GetFull(Guid id) { var fileName = GetFileName(id); using var stream = File.OpenRead(fileName); return CachedDocument.ReadFull(stream); } private void Process() { try { _processing = true; while (true) { Guid docID; lock (_processingLock) { docID = UncachedDocuments.Keys.FirstOrDefault(); if (docID == Guid.Empty) { _processing = false; break; } } var document = LoadDocument(docID); if (document is null) { Logger.Send(LogType.Error, "", $"Document {docID} cannot be cached since it does not exist."); } else { Add(document); } UncachedDocuments.TryRemove(docID, out var _); } } catch(Exception ex) { CoreUtils.LogException("", ex); _processing = false; } } private void CheckProcessing() { lock (_processingLock) { if (!_processing && UncachedDocuments.Any()) { Task.Run(Process); } } } private void EnsureCacheFolder() { Directory.CreateDirectory(GetFolder()); } private void LoadCurrentCache() { foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document")) { try { using var stream = File.OpenRead(file); var doc = CachedDocument.ReadHeader(stream); CachedDocuments.TryAdd(doc.ID, 0); } catch(Exception e) { CoreUtils.LogException("", e, "Error loading cache"); // Skip; } } } private string GetFolder() { return Path.Combine( CoreUtils.GetPath(), ClientFactory.DatabaseID.ToString(), "_documentcache", Tag); } private string GetFileName(Guid documentID) { return Path.Combine(GetFolder(), $"{documentID}.document"); } #endregion } public static class DocumentCaches { private static readonly Dictionary Caches = new Dictionary(); #region Registry public static void RegisterAll() { var types = CoreUtils.TypeList(x => !x.IsAbstract && !x.IsGenericType && x.IsSubclassOf(typeof(IDocumentCache))); foreach(var type in types) { Caches.Add(type, (Activator.CreateInstance(type) as IDocumentCache)!); } } public static void RegisterCache() where T : IDocumentCache, new() { Caches.Add(typeof(T), new T()); } public static T GetOrRegister() where T : class, IDocumentCache, new() { if(!Caches.TryGetValue(typeof(T), out var cache)) { cache = new T(); Caches.Add(typeof(T), new T()); } return (cache as T)!; } #endregion #region Interface public static void Clear() { foreach(var cache in Caches.Values) { cache.Clear(); } } public static void ClearOld() { foreach (var cache in Caches.Values) { cache.ClearOld(); } } #endregion } }