From 97fad301ae5563a82fb64fd9a537bde05334ac17 Mon Sep 17 00:00:00 2001 From: Mathew Sutcliffe Date: Tue, 2 Feb 2016 18:03:32 +0000 Subject: [PATCH] refs #581 Use JSON to store timestamps in data cache revision file. --- src/blackmisc/datacache.cpp | 98 +++++++++++++++++++++++++++++++----- src/blackmisc/datacache.h | 23 ++++++--- src/blackmisc/valuecache.cpp | 15 +++++- src/blackmisc/valuecache.h | 5 +- 4 files changed, 120 insertions(+), 21 deletions(-) diff --git a/src/blackmisc/datacache.cpp b/src/blackmisc/datacache.cpp index 52feb33ce..d22e20ced 100644 --- a/src/blackmisc/datacache.cpp +++ b/src/blackmisc/datacache.cpp @@ -103,7 +103,7 @@ namespace BlackMisc void CDataCache::saveToStoreAsync(const BlackMisc::CValueCachePacket &values) { - auto baseline = getAllValues(); + auto baseline = getAllValuesWithTimestamps(); QTimer::singleShot(0, &m_serializer, [this, baseline, values] { m_serializer.saveToStore(values.toVariantMap(), baseline); @@ -112,7 +112,7 @@ namespace BlackMisc void CDataCache::loadFromStoreAsync() { - auto baseline = getAllValues(); + auto baseline = getAllValuesWithTimestamps(); QTimer::singleShot(0, &m_serializer, [this, baseline] { m_serializer.loadFromStore(baseline); @@ -130,14 +130,15 @@ namespace BlackMisc return m_cache->persistentStore(); } - void CDataCacheSerializer::saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CVariantMap &baseline) + void CDataCacheSerializer::saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CValueCachePacket &baseline) { m_cache->m_revision.notifyPendingWrite(); auto lock = loadFromStore(baseline, true); // last-minute check for remote changes before clobbering the revision file for (const auto &key : values.keys()) { m_deferredChanges.remove(key); } // ignore changes that we are about to overwrite if (! lock) { return; } - m_cache->m_revision.writeNewRevision(); + m_cache->m_revision.writeNewRevision(baseline.toTimestampMap()); + m_cache->saveToFiles(persistentStore(), values); if (! m_deferredChanges.isEmpty()) // apply changes which we grabbed at the last minute above @@ -148,13 +149,13 @@ namespace BlackMisc } } - CDataCacheRevision::LockGuard CDataCacheSerializer::loadFromStore(const BlackMisc::CVariantMap &baseline, bool defer) + CDataCacheRevision::LockGuard CDataCacheSerializer::loadFromStore(const BlackMisc::CValueCachePacket &baseline, bool defer) { - auto lock = m_cache->m_revision.beginUpdate(); + auto lock = m_cache->m_revision.beginUpdate(baseline.toTimestampMap()); if (lock && m_cache->m_revision.isPendingRead()) { CValueCachePacket newValues; - m_cache->loadFromFiles(persistentStore(), baseline, newValues); + m_cache->loadFromFiles(persistentStore(), m_cache->m_revision.keysWithNewerTimestamps(), baseline.toVariantMap(), newValues); m_deferredChanges.insert(newValues); } @@ -167,7 +168,7 @@ namespace BlackMisc return lock; } - CDataCacheRevision::LockGuard CDataCacheRevision::beginUpdate() + CDataCacheRevision::LockGuard CDataCacheRevision::beginUpdate(const QMap ×tamps) { QMutexLocker lock(&m_mutex); @@ -182,6 +183,8 @@ namespace BlackMisc m_updateInProgress = true; LockGuard guard(this); + m_timestamps.clear(); + QFile revisionFile(m_basename + "/.rev"); if (revisionFile.exists()) { @@ -191,9 +194,35 @@ namespace BlackMisc return {}; } - QUuid uuid(revisionFile.readAll()); - if (uuid == m_uuid) + auto json = QJsonDocument::fromJson(revisionFile.readAll()).object(); + if (json.contains("uuid") && json.contains("timestamps")) { + QUuid uuid(json.value("uuid").toString()); + if (uuid == m_uuid) + { + if (m_pendingWrite) { return guard; } + return {}; + } + m_uuid = uuid; + + auto newTimestamps = fromJson(json.value("timestamps").toObject()); + for (auto it = newTimestamps.cbegin(); it != newTimestamps.cend(); ++it) + { + if (timestamps.value(it.key(), 0) < it.value()) + { + m_timestamps.insert(it.key(), it.value()); + } + } + if (m_timestamps.isEmpty()) + { + if (m_pendingWrite) { return guard; } + return {}; + } + } + else if (revisionFile.size() > 0) + { + CLogMessage(this).error("Invalid format of %1") << revisionFile.fileName(); + if (m_pendingWrite) { return guard; } return {}; } @@ -203,11 +232,12 @@ namespace BlackMisc return guard; } - void CDataCacheRevision::writeNewRevision() + void CDataCacheRevision::writeNewRevision(const QMap &i_timestamps) { QMutexLocker lock(&m_mutex); Q_ASSERT(m_updateInProgress); + Q_ASSERT(m_pendingWrite); Q_ASSERT(m_lockFile.isLocked()); QFile revisionFile(m_basename + "/.rev"); @@ -218,7 +248,16 @@ namespace BlackMisc } m_uuid = CIdentifier().toUuid(); - revisionFile.write(m_uuid.toByteArray()); + auto timestamps = m_timestamps; + for (auto it = i_timestamps.cbegin(); it != i_timestamps.cend(); ++it) + { + timestamps.insert(it.key(), it.value()); + } + + QJsonObject json; + json.insert("uuid", m_uuid.toString()); + json.insert("timestamps", toJson(timestamps)); + revisionFile.write(QJsonDocument(json).toJson()); } void CDataCacheRevision::finishUpdate() @@ -246,4 +285,39 @@ namespace BlackMisc { m_pendingWrite = true; } + + QSet CDataCacheRevision::keysWithNewerTimestamps() const + { + QMutexLocker lock(&m_mutex); + + Q_ASSERT(m_updateInProgress); + return QSet::fromList(m_timestamps.keys()); + } + + bool CDataCacheRevision::isNewerValueAvailable(const QString &key) const + { + QMutexLocker lock(&m_mutex); + + return m_updateInProgress && m_timestamps.contains(key); + } + + QJsonObject CDataCacheRevision::toJson(const QMap ×tamps) + { + QJsonObject result; + for (auto it = timestamps.begin(); it != timestamps.end(); ++it) + { + result.insert(it.key(), it.value()); + } + return result; + } + + QMap CDataCacheRevision::fromJson(const QJsonObject ×tamps) + { + QMap result; + for (auto it = timestamps.begin(); it != timestamps.end(); ++it) + { + result.insert(it.key(), static_cast(it.value().toDouble())); + } + return result; + } } diff --git a/src/blackmisc/datacache.h b/src/blackmisc/datacache.h index 44f844e79..023d2ce10 100644 --- a/src/blackmisc/datacache.h +++ b/src/blackmisc/datacache.h @@ -42,12 +42,13 @@ namespace BlackMisc //! RAII class to keep the revision file locked during update. class LockGuard; - //! Get the state of the disk cache, and prepare to update values. + //! Get the state of the disk cache, and prepare to update any values which are out of date. //! Return value can be converted to bool, false means update is not started (error, or already up-to-date). - LockGuard beginUpdate(); + //! \param timestamps Current in-memory timestamps, to be compared with the on-disk ones. + LockGuard beginUpdate(const QMap ×tamps); - //! During update, writes a new revision file. - void writeNewRevision(); + //! During update, writes a new revision file with new timestamps. + void writeNewRevision(const QMap ×tamps); //! Release the revision file lock and mark everything up-to-date (called by LockGuard destructor). void finishUpdate(); @@ -58,6 +59,12 @@ namespace BlackMisc //! Call before beginUpdate if there is a write pending, so update will start even if there is nothing to read. void notifyPendingWrite(); + //! During update, returns keys which have on-disk timestamps newer than in-memory. Guaranteed not empty. + QSet keysWithNewerTimestamps() const; + + //! During update, returns true if the on-disk timestamp of this key is newer than in-memory. + bool isNewerValueAvailable(const QString &key) const; + private: mutable QMutex m_mutex { QMutex::Recursive }; bool m_updateInProgress = false; @@ -66,6 +73,10 @@ namespace BlackMisc QString m_basename; QLockFile m_lockFile { m_basename + "/.lock" }; QUuid m_uuid; + QMap m_timestamps; + + static QJsonObject toJson(const QMap ×tamps); + static QMap fromJson(const QJsonObject ×tamps); }; /*! @@ -81,14 +92,14 @@ namespace BlackMisc CDataCacheSerializer(CDataCache *owner, const QString &revisionFileName); //! Save values to persistent store. Called whenever a value is changed locally. - void saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CVariantMap &baseline); + void saveToStore(const BlackMisc::CVariantMap &values, const BlackMisc::CValueCachePacket &baseline); //! Load values from persistent store. Called once per second. //! Also called by saveToStore, to ensure that remote changes to unrelated values are not lost. //! \param baseline A snapshot of the currently loaded values, taken when the load is queued. //! \param defer Whether to defer applying the changes. Used when called by saveToStore. //! \return Usually ignored, but can be held in order to retain the revision file lock. - CDataCacheRevision::LockGuard loadFromStore(const BlackMisc::CVariantMap &baseline, bool defer = false); + CDataCacheRevision::LockGuard loadFromStore(const BlackMisc::CValueCachePacket &baseline, bool defer = false); signals: //! Signal back to the cache when values have been loaded. diff --git a/src/blackmisc/valuecache.cpp b/src/blackmisc/valuecache.cpp index c0a05b1ce..17b1329d5 100644 --- a/src/blackmisc/valuecache.cpp +++ b/src/blackmisc/valuecache.cpp @@ -62,6 +62,16 @@ namespace BlackMisc return result; } + QMap CValueCachePacket::toTimestampMap() const + { + QMap result; + for (auto it = cbegin(); it != cend(); ++it) + { + implementationOf(result).insert(result.cend(), it.key(), it.timestamp()); + } + return result; + } + void CValueCachePacket::registerMetadata() { MetaType::registerMetadata(); @@ -259,13 +269,13 @@ namespace BlackMisc { QMutexLocker lock(&m_mutex); CValueCachePacket values; - auto status = loadFromFiles(dir, getAllValues(), values); + auto status = loadFromFiles(dir, {}, getAllValues(), values); values.setSaved(); insertValues(values); return status; } - CStatusMessage CValueCache::loadFromFiles(const QString &dir, const CVariantMap ¤tValues, CValueCachePacket &o_values) const + CStatusMessage CValueCache::loadFromFiles(const QString &dir, const QSet &keys, const CVariantMap ¤tValues, CValueCachePacket &o_values) const { if (! QDir(dir).isReadable()) { @@ -285,6 +295,7 @@ namespace BlackMisc } CVariantMap temp; temp.convertFromJson(json.object()); + temp.removeByKeyIf([&keys](const QString &key) { return keys.contains(key); }); // TODO optimize by skipping files temp.removeDuplicates(currentValues); o_values.insert(temp, QFileInfo(file).lastModified().toMSecsSinceEpoch()); } diff --git a/src/blackmisc/valuecache.h b/src/blackmisc/valuecache.h index 72fda9a86..dec46533d 100644 --- a/src/blackmisc/valuecache.h +++ b/src/blackmisc/valuecache.h @@ -59,6 +59,9 @@ namespace BlackMisc //! Discard timestamps and return as variant map. CVariantMap toVariantMap() const; + //! Discard values and return as map of timestamps. + QMap toTimestampMap() const; + //! \copydoc CValueObject::registerMetadata` static void registerMetadata(); @@ -207,7 +210,7 @@ namespace BlackMisc //! Load from Json files in a given directory any values which differ from the current ones, and insert them in o_values. //! \threadsafe - CStatusMessage loadFromFiles(const QString &directory, const CVariantMap ¤t, CValueCachePacket &o_values) const; + CStatusMessage loadFromFiles(const QString &directory, const QSet &keys, const CVariantMap ¤t, CValueCachePacket &o_values) const; //! Mark all values with keys that start with the given prefix as having been saved. //! \threadsafe