DocumentCache.cs 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566
  1. using InABox.Clients;
  2. using System;
  3. using System.Collections.Concurrent;
  4. using System.Collections.Generic;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Security.Cryptography;
  8. using System.Text;
  9. using System.Threading.Tasks;
  10. namespace InABox.Core
  11. {
  12. public interface ICachedDocument
  13. {
  14. Guid ID { get; }
  15. public void SerializeBinary(CoreBinaryWriter writer);
  16. void DeserializeBinary(CoreBinaryReader reader, bool full);
  17. }
  18. public class CachedDocument<T> : ISerializeBinary
  19. where T: ICachedDocument, new()
  20. {
  21. public Guid ID { get; set; }
  22. public DateTime CachedAt { get; set; }
  23. public T Document { get; set; }
  24. private CachedDocument()
  25. {
  26. }
  27. public CachedDocument(T document)
  28. {
  29. if (document.ID == Guid.Empty)
  30. {
  31. throw new Exception("Cannot cache document with no ID");
  32. }
  33. ID = document.ID;
  34. CachedAt = DateTime.Now;
  35. Document = document;
  36. }
  37. public void SerializeBinary(CoreBinaryWriter writer)
  38. {
  39. writer.Write(ID);
  40. writer.Write(CachedAt);
  41. Document.SerializeBinary(writer);
  42. }
  43. private void DeserializeBinary(CoreBinaryReader reader, bool full)
  44. {
  45. ID = reader.ReadGuid();
  46. CachedAt = reader.ReadDateTime();
  47. Document = new T();
  48. Document.DeserializeBinary(reader, full);
  49. }
  50. public void DeserializeBinary(CoreBinaryReader reader)
  51. {
  52. DeserializeBinary(reader, true);
  53. }
  54. public static CachedDocument<T> ReadHeader(Stream stream)
  55. {
  56. var cache = new CachedDocument<T>();
  57. cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: false);
  58. return cache;
  59. }
  60. public static CachedDocument<T> ReadFull(Stream stream)
  61. {
  62. var cache = new CachedDocument<T>();
  63. cache.DeserializeBinary(new CoreBinaryReader(stream, BinarySerializationSettings.Latest), full: true);
  64. return cache;
  65. }
  66. }
  67. public interface IDocumentCache
  68. {
  69. void Clear();
  70. void ClearOld();
  71. }
  72. /// <summary>
  73. /// A cache of documents that is saved in the AppData folder under a specific tag. Contrary to the name, this isn't just a <see cref="Document"/> cache, but
  74. /// in fact a way to cache any kind of object that implements <see cref="ICachedDocument"/>.
  75. /// </summary>
  76. /// <remarks>
  77. /// The files are stored with the name "&lt;ID&gt;.document", and they contain a binary serialised <see cref="CachedDocument"/>.
  78. /// This stores the date when the document was cached, allowing us to clear out old documents.
  79. /// </remarks>
  80. public abstract class DocumentCache<T> : IDocumentCache
  81. where T : class, ICachedDocument, new()
  82. {
  83. public string Tag { get; set; }
  84. public ConcurrentDictionary<Guid, byte> CachedDocuments { get; set; } = new ConcurrentDictionary<Guid, byte>();
  85. public ConcurrentDictionary<Guid, byte> UncachedDocuments { get; set; } = new ConcurrentDictionary<Guid, byte>();
  86. private bool _processing = false;
  87. private object _processingLock = new object();
  88. /// <summary>
  89. /// How long before documents are allowed to be cleaned.
  90. /// </summary>
  91. public abstract TimeSpan MaxAge { get; }
  92. public DocumentCache(string tag)
  93. {
  94. Tag = tag;
  95. EnsureCacheFolder();
  96. LoadCurrentCache();
  97. }
  98. #region Abstract Interface
  99. protected abstract T? LoadDocument(Guid id);
  100. #endregion
  101. #region Public Interface
  102. /// <summary>
  103. /// Check if the cache contains a document with the given <paramref name="id"/>.
  104. /// </summary>
  105. /// <param name="id"></param>
  106. /// <returns></returns>
  107. public bool Has(Guid id)
  108. {
  109. return CachedDocuments.ContainsKey(id);
  110. }
  111. /// <summary>
  112. /// Fetch a document from the cache or, if it hasn't been cached, through <see cref="LoadDocument(Guid)"/>.
  113. /// </summary>
  114. /// <param name="id"></param>
  115. /// <returns><see langword="null"/> if <see cref="LoadDocument(Guid)"/> returned <see langword="null"/>.</returns>
  116. public T? GetDocument(Guid id)
  117. {
  118. var file = GetFileName(id);
  119. if (File.Exists(file))
  120. {
  121. using var stream = File.OpenRead(file);
  122. var doc = CachedDocument<T>.ReadFull(stream);
  123. return doc.Document;
  124. }
  125. var document = LoadDocument(id);
  126. if(document != null)
  127. {
  128. Add(document);
  129. }
  130. return document;
  131. }
  132. /// <summary>
  133. /// Add a loaded document to the cache.
  134. /// </summary>
  135. /// <param name="document"></param>
  136. public void Add(T document)
  137. {
  138. var cached = new CachedDocument<T>(document);
  139. using (var file = File.Open(GetFileName(cached.ID), FileMode.Create))
  140. {
  141. cached.WriteBinary(file, BinarySerializationSettings.Latest);
  142. }
  143. CachedDocuments.TryAdd(cached.ID, 0);
  144. }
  145. /// <summary>
  146. /// Remove a document from the cache.
  147. /// </summary>
  148. /// <param name="id"></param>
  149. public void Remove(Guid id)
  150. {
  151. File.Delete(GetFileName(id));
  152. UncachedDocuments.TryRemove(id, out var _);
  153. CachedDocuments.TryRemove(id, out _);
  154. }
  155. /// <summary>
  156. /// Ensure that the cache contains <paramref name="documentIDs"/>. If it does not, a background worker will begin to download them.
  157. /// </summary>
  158. /// <param name="documentIDs"></param>
  159. public void Ensure(IEnumerable<Guid> documentIDs)
  160. {
  161. foreach(var docID in documentIDs)
  162. {
  163. if (docID != Guid.Empty)
  164. {
  165. if(!CachedDocuments.ContainsKey(docID))
  166. {
  167. UncachedDocuments.TryAdd(docID, 0);
  168. }
  169. }
  170. }
  171. CheckProcessing();
  172. }
  173. /// <summary>
  174. /// Like <see cref="Ensure(IEnumerable{Guid})"/>, but will clear out items that are not in <paramref name="documentIDs"/>.
  175. /// </summary>
  176. /// <param name="documentIDs"></param>
  177. public void EnsureStrict(IList<Guid> documentIDs)
  178. {
  179. foreach (var docID in documentIDs)
  180. {
  181. if (docID != Guid.Empty)
  182. {
  183. if (!CachedDocuments.ContainsKey(docID))
  184. {
  185. UncachedDocuments.TryAdd(docID, 0);
  186. }
  187. }
  188. }
  189. ClearWhere(x => !documentIDs.Contains(x));
  190. CheckProcessing();
  191. }
  192. /// <summary>
  193. /// Clear all old cached documents, according to <see cref="MaxAge"/>.
  194. /// </summary>
  195. public void ClearOld()
  196. {
  197. ClearWhere(docID =>
  198. {
  199. var filename = GetFileName(docID);
  200. if (File.Exists(filename))
  201. {
  202. using var stream = File.OpenRead(filename);
  203. var doc = CachedDocument<T>.ReadHeader(stream);
  204. return DateTime.Now - doc.CachedAt > MaxAge;
  205. }
  206. else
  207. {
  208. return true;
  209. }
  210. });
  211. }
  212. /// <summary>
  213. /// Clear the entire cache.
  214. /// </summary>
  215. public void Clear()
  216. {
  217. foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document"))
  218. {
  219. File.Delete(file);
  220. }
  221. CachedDocuments.Clear();
  222. UncachedDocuments.Clear();
  223. }
  224. #endregion
  225. #region Private Methods
  226. private void ClearWhere(Func<Guid, bool> predicate)
  227. {
  228. var toRemove = new List<Guid>();
  229. foreach (var docID in CachedDocuments.Keys)
  230. {
  231. if (predicate(docID))
  232. {
  233. File.Delete(GetFileName(docID));
  234. toRemove.Add(docID);
  235. }
  236. }
  237. foreach (var id in toRemove)
  238. {
  239. CachedDocuments.TryRemove(id, out var _);
  240. }
  241. }
  242. protected CachedDocument<T> GetHeader(Guid id)
  243. {
  244. var fileName = GetFileName(id);
  245. using var stream = File.OpenRead(fileName);
  246. return CachedDocument<T>.ReadHeader(stream);
  247. }
  248. protected CachedDocument<T> GetFull(Guid id)
  249. {
  250. var fileName = GetFileName(id);
  251. using var stream = File.OpenRead(fileName);
  252. return CachedDocument<T>.ReadFull(stream);
  253. }
  254. private void Process()
  255. {
  256. try
  257. {
  258. _processing = true;
  259. while (true)
  260. {
  261. Guid docID;
  262. lock (_processingLock)
  263. {
  264. docID = UncachedDocuments.Keys.FirstOrDefault();
  265. if (docID == Guid.Empty)
  266. {
  267. _processing = false;
  268. break;
  269. }
  270. }
  271. var document = LoadDocument(docID);
  272. if (document is null)
  273. {
  274. Logger.Send(LogType.Error, "", $"Document {docID} cannot be cached since it does not exist.");
  275. }
  276. else
  277. {
  278. Add(document);
  279. }
  280. UncachedDocuments.TryRemove(docID, out var _);
  281. }
  282. }
  283. catch(Exception ex)
  284. {
  285. CoreUtils.LogException("", ex);
  286. _processing = false;
  287. }
  288. }
  289. private void CheckProcessing()
  290. {
  291. lock (_processingLock)
  292. {
  293. if (!_processing && UncachedDocuments.Any())
  294. {
  295. Task.Run(Process);
  296. }
  297. }
  298. }
  299. private void EnsureCacheFolder()
  300. {
  301. Directory.CreateDirectory(GetFolder());
  302. }
  303. private void LoadCurrentCache()
  304. {
  305. foreach (var file in Directory.EnumerateFiles(GetFolder()).Where(x => Path.GetExtension(x) == ".document"))
  306. {
  307. try
  308. {
  309. using var stream = File.OpenRead(file);
  310. var doc = CachedDocument<T>.ReadHeader(stream);
  311. CachedDocuments.TryAdd(doc.ID, 0);
  312. }
  313. catch(Exception e)
  314. {
  315. CoreUtils.LogException("", e, "Error loading cache");
  316. // Skip;
  317. }
  318. }
  319. }
  320. private string GetFolder()
  321. {
  322. return Path.Combine(
  323. CoreUtils.GetPath(),
  324. ClientFactory.DatabaseID.ToString(),
  325. "_documentcache",
  326. Tag);
  327. }
  328. private string GetFileName(Guid documentID)
  329. {
  330. return Path.Combine(GetFolder(), $"{documentID}.document");
  331. }
  332. #endregion
  333. }
  334. public static class DocumentCaches
  335. {
  336. private static readonly Dictionary<Type, IDocumentCache> Caches = new Dictionary<Type, IDocumentCache>();
  337. #region Registry
  338. public static void RegisterAll()
  339. {
  340. var types = CoreUtils.TypeList(x => !x.IsAbstract && !x.IsGenericType && x.IsSubclassOf(typeof(IDocumentCache)));
  341. foreach(var type in types)
  342. {
  343. Caches.Add(type, (Activator.CreateInstance(type) as IDocumentCache)!);
  344. }
  345. }
  346. public static void RegisterCache<T>()
  347. where T : IDocumentCache, new()
  348. {
  349. Caches.Add(typeof(T), new T());
  350. }
  351. public static T GetOrRegister<T>()
  352. where T : class, IDocumentCache, new()
  353. {
  354. if(!Caches.TryGetValue(typeof(T), out var cache))
  355. {
  356. cache = new T();
  357. Caches.Add(typeof(T), new T());
  358. }
  359. return (cache as T)!;
  360. }
  361. #endregion
  362. #region Interface
  363. public static void Clear()
  364. {
  365. foreach(var cache in Caches.Values)
  366. {
  367. cache.Clear();
  368. }
  369. }
  370. public static void ClearOld()
  371. {
  372. foreach (var cache in Caches.Values)
  373. {
  374. cache.ClearOld();
  375. }
  376. }
  377. #endregion
  378. }
  379. /// <summary>
  380. /// An implementation of <see cref="ICachedDocument"/> for use with entities of type <see cref="Document"/>. The <see cref="Document.TimeStamp"/>
  381. /// is saved along with the document, allowing us to refresh updated documents.
  382. /// </summary>
  383. public class DocumentCachedDocument : ICachedDocument
  384. {
  385. public DateTime TimeStamp { get; set; }
  386. public Document? Document { get; set; }
  387. public Guid ID => Document?.ID ?? Guid.Empty;
  388. public DocumentCachedDocument() { }
  389. public DocumentCachedDocument(Document document)
  390. {
  391. Document = document;
  392. TimeStamp = document.TimeStamp;
  393. }
  394. public void DeserializeBinary(CoreBinaryReader reader, bool full)
  395. {
  396. TimeStamp = reader.ReadDateTime();
  397. if (full)
  398. {
  399. Document = reader.ReadObject<Document>();
  400. }
  401. }
  402. public void SerializeBinary(CoreBinaryWriter writer)
  403. {
  404. writer.Write(TimeStamp);
  405. if (Document is null)
  406. {
  407. throw new Exception("Cannot serialize incomplete CachedDocument");
  408. }
  409. writer.WriteObject(Document);
  410. }
  411. }
  412. /// <summary>
  413. /// Implements a <see cref="DocumentCache{T}"/> for use with <see cref="Document"/>.
  414. /// </summary>
  415. public abstract class DocumentCache : DocumentCache<DocumentCachedDocument>
  416. {
  417. public DocumentCache(string tag): base(tag) { }
  418. protected override DocumentCachedDocument? LoadDocument(Guid id)
  419. {
  420. var document = Client.Query(new Filter<Document>(x => x.ID).IsEqualTo(id))
  421. .ToObjects<Document>().FirstOrDefault();
  422. if(document != null)
  423. {
  424. return new DocumentCachedDocument(document);
  425. }
  426. else
  427. {
  428. return null;
  429. }
  430. }
  431. /// <summary>
  432. /// Fetch a bunch of documents from the cache or the database, optionally checking against the timestamp listed in the database.
  433. /// </summary>
  434. /// <param name="ids"></param>
  435. /// <param name="checkTimestamp">
  436. /// If <see langword="true"/>, then loads <see cref="Document.TimeStamp"/> from the database for all cached documents,
  437. /// and if they are older, updates the cache.
  438. /// </param>
  439. public IEnumerable<Document> LoadDocuments(IEnumerable<Guid> ids, bool checkTimestamp = false)
  440. {
  441. var cached = new List<Guid>();
  442. var toLoad = new List<Guid>();
  443. foreach (var docID in ids)
  444. {
  445. if (Has(docID))
  446. {
  447. cached.Add(docID);
  448. }
  449. else
  450. {
  451. toLoad.Add(docID);
  452. }
  453. }
  454. var loadedCached = new List<Document>();
  455. if (cached.Count > 0)
  456. {
  457. var docs = Client.Query(
  458. new Filter<Document>(x => x.ID).InList(cached.ToArray()),
  459. Columns.None<Document>().Add(x => x.TimeStamp, x => x.ID));
  460. foreach (var doc in docs.ToObjects<Document>())
  461. {
  462. try
  463. {
  464. var timestamp = GetHeader(doc.ID).Document.TimeStamp;
  465. if (doc.TimeStamp > timestamp)
  466. {
  467. toLoad.Add(doc.ID);
  468. }
  469. else
  470. {
  471. loadedCached.Add(GetFull(doc.ID).Document.Document!);
  472. }
  473. }
  474. catch (Exception e)
  475. {
  476. CoreUtils.LogException("", e, "Error loading cached file");
  477. toLoad.Add(doc.ID);
  478. }
  479. }
  480. }
  481. if (toLoad.Count > 0)
  482. {
  483. var loaded = Client.Query(new Filter<Document>(x => x.ID).InList(toLoad.ToArray()))
  484. .ToObjects<Document>().ToList();
  485. foreach (var loadedDoc in loaded)
  486. {
  487. Add(new DocumentCachedDocument(loadedDoc));
  488. }
  489. return loaded.Concat(loadedCached);
  490. }
  491. else
  492. {
  493. return loadedCached;
  494. }
  495. }
  496. }
  497. }