refs #536 Moved CDataCache serialization into a worker thread.

src/blackcore/datacache.cpp

@@ -26,12 +26,14 @@ namespace BlackCore
             CLogMessage(this).error("Failed to create directory %1") << persistentStore();
         }
 
-        connect(this, &CValueCache::valuesChangedByLocal, this, [this](const CValueCachePacket &values) { saveToStore(values.toVariantMap()); });
+        connect(this, &CValueCache::valuesChangedByLocal, this, &CDataCache::saveToStoreAsync);
-        connect(&m_watcher, &QFileSystemWatcher::fileChanged, this, [this] { loadFromStore(); });
+        connect(&m_watcher, &QFileSystemWatcher::fileChanged, this, &CDataCache::loadFromStoreAsync);
+        connect(&m_serializer, &CDataCacheSerializer::valuesLoadedFromStore, this, &CDataCache::changeValuesFromRemote);
 
         if (! QFile::exists(m_revisionFileName)) { QFile(m_revisionFileName).open(QFile::WriteOnly); }
         m_watcher.addPath(m_revisionFileName);
-        loadFromStore();
+        m_serializer.start();
+        loadFromStoreAsync();
     }
 
     CDataCache *CDataCache::instance()
@@ -74,10 +76,37 @@ namespace BlackCore
         }
     }
 
-    void CDataCache::saveToStore(const BlackMisc::CVariantMap &values)
+    void CDataCache::saveToStoreAsync(const BlackMisc::CValueCachePacket &values)
     {
-        QMutexLocker lock(&m_mutex);
+        auto baseline = getAllValues();
+        QTimer::singleShot(0, &m_serializer, [this, baseline, values]
+        {
+            m_serializer.saveToStore(values.toVariantMap(), baseline);
+        });
+    }
 
+    void CDataCache::loadFromStoreAsync()
+    {
+        auto baseline = getAllValues();
+        QTimer::singleShot(0, &m_serializer, [this, baseline]
+        {
+            m_serializer.loadFromStore(baseline);
+        });
+    }
+
+    CDataCacheSerializer::CDataCacheSerializer(CDataCache *owner, const QString &revisionFileName) :
+        CContinuousWorker(owner),
+        m_cache(owner),
+        m_revisionFileName(revisionFileName)
+    {}
+
+    const QString &CDataCacheSerializer::persistentStore() const
+    {
+        return m_cache->persistentStore();
+    }
+
+    void CDataCacheSerializer::saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CVariantMap &baseline)
+    {
         QLockFile revisionFileLock(m_revisionFileName + ".lock");
         if (! revisionFileLock.lock())
         {
@@ -85,7 +114,7 @@ namespace BlackCore
             return;
         }
 
-        loadFromStore(false, true); // last-minute check for remote changes before clobbering the revision file
+        loadFromStore(baseline, false, true); // last-minute check for remote changes before clobbering the revision file
         for (const auto &key : values.keys()) { m_deferredChanges.remove(key); } // ignore changes that we are about to overwrite
 
         QFile revisionFile(m_revisionFileName);
@@ -97,13 +126,17 @@ namespace BlackCore
         m_revision = CIdentifier().toUuid();
         revisionFile.write(m_revision.toByteArray());
 
-        saveToFiles(persistentStore(), values);
+        m_cache->saveToFiles(persistentStore(), values);
+
+        if (! m_deferredChanges.isEmpty()) // apply changes which we grabbed at the last minute above
+        {
+            emit valuesLoadedFromStore(m_deferredChanges, CIdentifier::anonymous());
+            m_deferredChanges.clear();
+        }
     }
 
-    void CDataCache::loadFromStore(bool revLock, bool defer)
+    void CDataCacheSerializer::loadFromStore(const BlackMisc::CVariantMap &baseline, bool revLock, bool defer)
     {
-        QMutexLocker lock(&m_mutex);
-
         QLockFile revisionFileLock(m_revisionFileName + ".lock");
         if (revLock && ! revisionFileLock.lock())
         {
@@ -127,13 +160,13 @@ namespace BlackCore
         {
             m_revision = newRevision;
             CValueCachePacket newValues;
-            loadFromFiles(persistentStore(), newValues);
+            m_cache->loadFromFiles(persistentStore(), baseline, newValues);
             m_deferredChanges.insert(newValues);
         }
 
         if (! (m_deferredChanges.isEmpty() || defer))
         {
-            changeValuesFromRemote(m_deferredChanges, CIdentifier::anonymous());
+            emit valuesLoadedFromStore(m_deferredChanges, CIdentifier::anonymous());
             m_deferredChanges.clear();
         }
     }

src/blackcore/datacache.h

@@ -14,12 +14,50 @@
 
 #include "blackcore/blackcoreexport.h"
 #include "blackmisc/valuecache.h"
+#include "blackmisc/worker.h"
 #include <QUuid>
 #include <QFileSystemWatcher>
 
 namespace BlackCore
 {
 
+    class CDataCache;
+
+    /*!
+     * Worker which performs (de)serialization on behalf of CDataCache, in a separate thread
+     * so that the main thread is not blocked by (de)serialization of large objects.
+     */
+    class BLACKCORE_EXPORT CDataCacheSerializer : public BlackMisc::CContinuousWorker
+    {
+        Q_OBJECT
+
+    public:
+        //! Constructor.
+        CDataCacheSerializer(CDataCache *owner, const QString &revisionFileName);
+
+        //! Save values to persistent store. Called whenever a value is changed locally.
+        void saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CVariantMap &baseline);
+
+        //! Load values from persistent store. Called once per second.
+        //! Also called by saveToStore, to ensure that remote changes to unrelated values are not lost.
+        //! \param baseline A snapshot of the currently loaded values, taken when the load is queued.
+        //! \param lock Whether to acquire the revision file lock. Used when called by saveToStore.
+        //! \param defer Whether to defer applying the changes. Used when called by saveToStore.
+        void loadFromStore(const BlackMisc::CVariantMap &baseline, bool lock = true, bool defer = false);
+
+    signals:
+        //! Signal back to the cache when values have been loaded.
+        void valuesLoadedFromStore(const BlackMisc::CValueCachePacket &values, const BlackMisc::CIdentifier &originator);
+
+    private:
+        const QString &persistentStore() const;
+
+        const CDataCache *const m_cache = nullptr;
+        QUuid m_revision;
+        const QString m_revisionFileName;
+        BlackMisc::CValueCachePacket m_deferredChanges;
+    };
+
     /*!
      * Singleton derived class of CValueCache, for core dynamic data.
      *
@@ -45,19 +83,14 @@ namespace BlackCore
     private:
         CDataCache();
 
-        //! Save values to persistent store. Called whenever a value is changed locally.
+        void saveToStoreAsync(const BlackMisc::CValueCachePacket &values);
-        void saveToStore(const BlackMisc::CVariantMap &values);
+        void loadFromStoreAsync();
-
-        //! Load values from persistent store. Called once per second.
-        //! Also called by saveToStore, to ensure that remote changes to unrelated values are not lost.
-        //! \param lock Whether to acquire the revision file lock. Used when called by saveToStore.
-        //! \param defer Whether to defer applying the changes. Used when called by saveToStore.
-        void loadFromStore(bool lock = true, bool defer = false);
 
         QFileSystemWatcher m_watcher;
-        QUuid m_revision;
         const QString m_revisionFileName { persistentStore() + "/.rev" };
-        BlackMisc::CValueCachePacket m_deferredChanges;
+
+        CDataCacheSerializer m_serializer { this, m_revisionFileName };
+        friend class CDataCacheSerializer; // to access protected members of CValueCache
     };
 
     /*!

src/blackmisc/valuecache.cpp

@@ -249,19 +249,17 @@ namespace BlackMisc
     {
         QMutexLocker lock(&m_mutex);
         CValueCachePacket values;
-        auto status = loadFromFiles(dir, values);
+        auto status = loadFromFiles(dir, getAllValues(), values);
         insertValues(values);
         return status;
     }
 
-    CStatusMessage CValueCache::loadFromFiles(const QString &dir, CValueCachePacket &o_values) const
+    CStatusMessage CValueCache::loadFromFiles(const QString &dir, const CVariantMap &currentValues, CValueCachePacket &o_values) const
     {
-        QMutexLocker lock(&m_mutex);
         if (! QDir(dir).isReadable())
         {
             return CLogMessage(this).error("Failed to read directory %1") << dir;
         }
-        auto currentValues = getAllValues();
         for (const auto &filename : QDir(dir).entryList({ "*.json" }, QDir::Files))
         {
             QFile file(dir + "/" + filename);

src/blackmisc/valuecache.h

@@ -162,10 +162,12 @@ namespace BlackMisc
 
     protected:
         //! Save specific values to Json files in a given directory.
+        //! \threadsafe
         CStatusMessage saveToFiles(const QString &directory, const CVariantMap &values) const;
 
         //! Load from Json files in a given directory any values which differ from the current ones, and insert them in o_values.
-        CStatusMessage loadFromFiles(const QString &directory, CValueCachePacket &o_values) const;
+        //! \threadsafe
+        CStatusMessage loadFromFiles(const QString &directory, const CVariantMap &current, CValueCachePacket &o_values) const;
 
         //! Mutex protecting operations which are critical on m_elements.
         mutable QMutex m_mutex { QMutex::Recursive };