refs #581 Use JSON to store timestamps in data cache revision file.

src/blackmisc/datacache.cpp

@@ -103,7 +103,7 @@ namespace BlackMisc
 
     void CDataCache::saveToStoreAsync(const BlackMisc::CValueCachePacket &values)
     {
-        auto baseline = getAllValues();
+        auto baseline = getAllValuesWithTimestamps();
         QTimer::singleShot(0, &m_serializer, [this, baseline, values]
         {
             m_serializer.saveToStore(values.toVariantMap(), baseline);
@@ -112,7 +112,7 @@ namespace BlackMisc
 
     void CDataCache::loadFromStoreAsync()
     {
-        auto baseline = getAllValues();
+        auto baseline = getAllValuesWithTimestamps();
         QTimer::singleShot(0, &m_serializer, [this, baseline]
         {
             m_serializer.loadFromStore(baseline);
@@ -130,14 +130,15 @@ namespace BlackMisc
         return m_cache->persistentStore();
     }
 
-    void CDataCacheSerializer::saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CVariantMap &baseline)
+    void CDataCacheSerializer::saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CValueCachePacket &baseline)
     {
         m_cache->m_revision.notifyPendingWrite();
         auto lock = loadFromStore(baseline, true); // last-minute check for remote changes before clobbering the revision file
         for (const auto &key : values.keys()) { m_deferredChanges.remove(key); } // ignore changes that we are about to overwrite
 
         if (! lock) { return; }
-        m_cache->m_revision.writeNewRevision();
+        m_cache->m_revision.writeNewRevision(baseline.toTimestampMap());
+
         m_cache->saveToFiles(persistentStore(), values);
 
         if (! m_deferredChanges.isEmpty()) // apply changes which we grabbed at the last minute above
@@ -148,13 +149,13 @@ namespace BlackMisc
         }
     }
 
-    CDataCacheRevision::LockGuard CDataCacheSerializer::loadFromStore(const BlackMisc::CVariantMap &baseline, bool defer)
+    CDataCacheRevision::LockGuard CDataCacheSerializer::loadFromStore(const BlackMisc::CValueCachePacket &baseline, bool defer)
     {
-        auto lock = m_cache->m_revision.beginUpdate();
+        auto lock = m_cache->m_revision.beginUpdate(baseline.toTimestampMap());
         if (lock && m_cache->m_revision.isPendingRead())
         {
             CValueCachePacket newValues;
-            m_cache->loadFromFiles(persistentStore(), baseline, newValues);
+            m_cache->loadFromFiles(persistentStore(), m_cache->m_revision.keysWithNewerTimestamps(), baseline.toVariantMap(), newValues);
             m_deferredChanges.insert(newValues);
         }
 
@@ -167,7 +168,7 @@ namespace BlackMisc
         return lock;
     }
 
-    CDataCacheRevision::LockGuard CDataCacheRevision::beginUpdate()
+    CDataCacheRevision::LockGuard CDataCacheRevision::beginUpdate(const QMap<QString, qint64> &timestamps)
     {
         QMutexLocker lock(&m_mutex);
 
@@ -182,6 +183,8 @@ namespace BlackMisc
         m_updateInProgress = true;
         LockGuard guard(this);
 
+        m_timestamps.clear();
+
         QFile revisionFile(m_basename + "/.rev");
         if (revisionFile.exists())
         {
@@ -191,9 +194,35 @@ namespace BlackMisc
                 return {};
             }
 
-            QUuid uuid(revisionFile.readAll());
-            if (uuid == m_uuid)
+            auto json = QJsonDocument::fromJson(revisionFile.readAll()).object();
+            if (json.contains("uuid") && json.contains("timestamps"))
             {
+                QUuid uuid(json.value("uuid").toString());
+                if (uuid == m_uuid)
+                {
+                    if (m_pendingWrite) { return guard; }
+                    return {};
+                }
+                m_uuid = uuid;
+
+                auto newTimestamps = fromJson(json.value("timestamps").toObject());
+                for (auto it = newTimestamps.cbegin(); it != newTimestamps.cend(); ++it)
+                {
+                    if (timestamps.value(it.key(), 0) < it.value())
+                    {
+                        m_timestamps.insert(it.key(), it.value());
+                    }
+                }
+                if (m_timestamps.isEmpty())
+                {
+                    if (m_pendingWrite) { return guard; }
+                    return {};
+                }
+            }
+            else if (revisionFile.size() > 0)
+            {
+                CLogMessage(this).error("Invalid format of %1") << revisionFile.fileName();
+
                 if (m_pendingWrite) { return guard; }
                 return {};
             }
@@ -203,11 +232,12 @@ namespace BlackMisc
         return guard;
     }
 
-    void CDataCacheRevision::writeNewRevision()
+    void CDataCacheRevision::writeNewRevision(const QMap<QString, qint64> &i_timestamps)
     {
         QMutexLocker lock(&m_mutex);
 
         Q_ASSERT(m_updateInProgress);
+        Q_ASSERT(m_pendingWrite);
         Q_ASSERT(m_lockFile.isLocked());
 
         QFile revisionFile(m_basename + "/.rev");
@@ -218,7 +248,16 @@ namespace BlackMisc
         }
 
         m_uuid = CIdentifier().toUuid();
-        revisionFile.write(m_uuid.toByteArray());
+        auto timestamps = m_timestamps;
+        for (auto it = i_timestamps.cbegin(); it != i_timestamps.cend(); ++it)
+        {
+            timestamps.insert(it.key(), it.value());
+        }
+
+        QJsonObject json;
+        json.insert("uuid", m_uuid.toString());
+        json.insert("timestamps", toJson(timestamps));
+        revisionFile.write(QJsonDocument(json).toJson());
     }
 
     void CDataCacheRevision::finishUpdate()
@@ -246,4 +285,39 @@ namespace BlackMisc
     {
         m_pendingWrite = true;
     }
+
+    QSet<QString> CDataCacheRevision::keysWithNewerTimestamps() const
+    {
+        QMutexLocker lock(&m_mutex);
+
+        Q_ASSERT(m_updateInProgress);
+        return QSet<QString>::fromList(m_timestamps.keys());
+    }
+
+    bool CDataCacheRevision::isNewerValueAvailable(const QString &key) const
+    {
+        QMutexLocker lock(&m_mutex);
+
+        return m_updateInProgress && m_timestamps.contains(key);
+    }
+
+    QJsonObject CDataCacheRevision::toJson(const QMap<QString, qint64> &timestamps)
+    {
+        QJsonObject result;
+        for (auto it = timestamps.begin(); it != timestamps.end(); ++it)
+        {
+            result.insert(it.key(), it.value());
+        }
+        return result;
+    }
+
+    QMap<QString, qint64> CDataCacheRevision::fromJson(const QJsonObject &timestamps)
+    {
+        QMap<QString, qint64> result;
+        for (auto it = timestamps.begin(); it != timestamps.end(); ++it)
+        {
+            result.insert(it.key(), static_cast<qint64>(it.value().toDouble()));
+        }
+        return result;
+    }
 }

src/blackmisc/datacache.h

@@ -42,12 +42,13 @@ namespace BlackMisc
         //! RAII class to keep the revision file locked during update.
         class LockGuard;
 
-        //! Get the state of the disk cache, and prepare to update values.
+        //! Get the state of the disk cache, and prepare to update any values which are out of date.
         //! Return value can be converted to bool, false means update is not started (error, or already up-to-date).
-        LockGuard beginUpdate();
+        //! \param timestamps Current in-memory timestamps, to be compared with the on-disk ones.
+        LockGuard beginUpdate(const QMap<QString, qint64> &timestamps);
 
-        //! During update, writes a new revision file.
-        void writeNewRevision();
+        //! During update, writes a new revision file with new timestamps.
+        void writeNewRevision(const QMap<QString, qint64> &timestamps);
 
         //! Release the revision file lock and mark everything up-to-date (called by LockGuard destructor).
         void finishUpdate();
@@ -58,6 +59,12 @@ namespace BlackMisc
         //! Call before beginUpdate if there is a write pending, so update will start even if there is nothing to read.
         void notifyPendingWrite();
 
+        //! During update, returns keys which have on-disk timestamps newer than in-memory. Guaranteed not empty.
+        QSet<QString> keysWithNewerTimestamps() const;
+
+        //! During update, returns true if the on-disk timestamp of this key is newer than in-memory.
+        bool isNewerValueAvailable(const QString &key) const;
+
     private:
         mutable QMutex m_mutex { QMutex::Recursive };
         bool m_updateInProgress = false;
@@ -66,6 +73,10 @@ namespace BlackMisc
         QString m_basename;
         QLockFile m_lockFile { m_basename + "/.lock" };
         QUuid m_uuid;
+        QMap<QString, qint64> m_timestamps;
+
+        static QJsonObject toJson(const QMap<QString, qint64> &timestamps);
+        static QMap<QString, qint64> fromJson(const QJsonObject &timestamps);
     };
 
     /*!
@@ -81,14 +92,14 @@ namespace BlackMisc
         CDataCacheSerializer(CDataCache *owner, const QString &revisionFileName);
 
         //! Save values to persistent store. Called whenever a value is changed locally.
-        void saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CVariantMap &baseline);
+        void saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CValueCachePacket &baseline);
 
         //! Load values from persistent store. Called once per second.
         //! Also called by saveToStore, to ensure that remote changes to unrelated values are not lost.
         //! \param baseline A snapshot of the currently loaded values, taken when the load is queued.
         //! \param defer Whether to defer applying the changes. Used when called by saveToStore.
         //! \return Usually ignored, but can be held in order to retain the revision file lock.
-        CDataCacheRevision::LockGuard loadFromStore(const BlackMisc::CVariantMap &baseline, bool defer = false);
+        CDataCacheRevision::LockGuard loadFromStore(const BlackMisc::CValueCachePacket &baseline, bool defer = false);
 
     signals:
         //! Signal back to the cache when values have been loaded.

src/blackmisc/valuecache.cpp

@@ -62,6 +62,16 @@ namespace BlackMisc
         return result;
     }
 
+    QMap<QString, qint64> CValueCachePacket::toTimestampMap() const
+    {
+        QMap<QString, qint64> result;
+        for (auto it = cbegin(); it != cend(); ++it)
+        {
+            implementationOf(result).insert(result.cend(), it.key(), it.timestamp());
+        }
+        return result;
+    }
+
     void CValueCachePacket::registerMetadata()
     {
         MetaType::registerMetadata();
@@ -259,13 +269,13 @@ namespace BlackMisc
     {
         QMutexLocker lock(&m_mutex);
         CValueCachePacket values;
-        auto status = loadFromFiles(dir, getAllValues(), values);
+        auto status = loadFromFiles(dir, {}, getAllValues(), values);
         values.setSaved();
         insertValues(values);
         return status;
     }
 
-    CStatusMessage CValueCache::loadFromFiles(const QString &dir, const CVariantMap &currentValues, CValueCachePacket &o_values) const
+    CStatusMessage CValueCache::loadFromFiles(const QString &dir, const QSet<QString> &keys, const CVariantMap &currentValues, CValueCachePacket &o_values) const
     {
         if (! QDir(dir).isReadable())
         {
@@ -285,6 +295,7 @@ namespace BlackMisc
             }
             CVariantMap temp;
             temp.convertFromJson(json.object());
+            temp.removeByKeyIf([&keys](const QString &key) { return keys.contains(key); }); // TODO optimize by skipping files
             temp.removeDuplicates(currentValues);
             o_values.insert(temp, QFileInfo(file).lastModified().toMSecsSinceEpoch());
         }

src/blackmisc/valuecache.h

@@ -59,6 +59,9 @@ namespace BlackMisc
         //! Discard timestamps and return as variant map.
         CVariantMap toVariantMap() const;
 
+        //! Discard values and return as map of timestamps.
+        QMap<QString, qint64> toTimestampMap() const;
+
         //! \copydoc CValueObject::registerMetadata`
         static void registerMetadata();
 
@@ -207,7 +210,7 @@ namespace BlackMisc
 
         //! Load from Json files in a given directory any values which differ from the current ones, and insert them in o_values.
         //! \threadsafe
-        CStatusMessage loadFromFiles(const QString &directory, const CVariantMap &current, CValueCachePacket &o_values) const;
+        CStatusMessage loadFromFiles(const QString &directory, const QSet<QString> &keys, const CVariantMap &current, CValueCachePacket &o_values) const;
 
         //! Mark all values with keys that start with the given prefix as having been saved.
         //! \threadsafe