refs #460 CDataCache for dynamic (downloaded/generated) data with file-based distribution among processes.

src/blackcore/datacache.cpp

@@ -0,0 +1,129 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+#include "datacache.h"
+#include "blackmisc/logmessage.h"
+#include "blackmisc/identifier.h"
+#include <QStandardPaths>
+
+using namespace BlackMisc;
+
+namespace BlackCore
+{
+
+    CDataCache::CDataCache() :
+        CValueCache(CValueCache::Distributed)
+    {
+        if (! QDir::root().mkpath(persistentStore()))
+        {
+            CLogMessage(this).error("Failed to create directory %1") << persistentStore();
+        }
+
+        connect(this, &CValueCache::valuesChangedByLocal, this, &CDataCache::saveToStore);
+
+        connect(&m_reloadTimer, &QTimer::timeout, this, [this]() { loadFromStore(); });
+        m_reloadTimer.start(1000);
+        loadFromStore();
+    }
+
+    CDataCache *CDataCache::instance()
+    {
+        static CDataCache cache;
+        return &cache;
+    }
+
+    const QString &CDataCache::persistentStore()
+    {
+        static const QString dir = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/org.swift-project/data/cache/core";
+        return dir;
+    }
+
+    QString lockFileError(const QLockFile &lock)
+    {
+        switch (lock.error())
+        {
+        case QLockFile::NoError: return "No error";
+        case QLockFile::PermissionError: return "Insufficient permission";
+        case QLockFile::UnknownError: return "Unknown filesystem error";
+        case QLockFile::LockFailedError:
+        {
+            QString hostname, appname;
+            qint64 pid = 0;
+            lock.getLockInfo(&pid, &hostname, &appname);
+            return QString("Lock open in another process (%1 %2 on %3)").arg(hostname, QString::number(pid), appname);
+        }
+        default: return "Bad error number";
+        }
+    }
+
+    void CDataCache::saveToStore(const BlackMisc::CVariantMap &values)
+    {
+        QMutexLocker lock(&m_mutex);
+
+        QLockFile revisionFileLock(m_revisionFileName);
+        if (! revisionFileLock.lock())
+        {
+            CLogMessage(this).error("Failed to lock %1: %2") << m_revisionFileName << lockFileError(revisionFileLock);
+            return;
+        }
+
+        loadFromStore(false, true); // last-minute check for remote changes before clobbering the revision file
+        for (const auto &key : values.keys()) { m_deferredChanges.remove(key); } // ignore changes that we are about to overwrite
+
+        QFile revisionFile(m_revisionFileName);
+        if (! revisionFile.open(QFile::WriteOnly))
+        {
+            CLogMessage(this).error("Failed to open %1: %2") << m_revisionFileName << revisionFile.errorString();
+            return;
+        }
+        m_revision = CIdentifier().toUuid();
+        revisionFile.write(m_revision.toByteArray());
+
+        saveToFiles(persistentStore(), values);
+    }
+
+    void CDataCache::loadFromStore(bool revLock, bool defer)
+    {
+        QMutexLocker lock(&m_mutex);
+
+        QLockFile revisionFileLock(m_revisionFileName);
+        if (revLock && ! revisionFileLock.lock())
+        {
+            CLogMessage(this).error("Failed to lock %1: %2") << m_revisionFileName << lockFileError(revisionFileLock);
+            return;
+        }
+
+        QFile revisionFile(m_revisionFileName);
+        if (! revisionFile.exists())
+        {
+            return;
+        }
+        if (! revisionFile.open(QFile::ReadOnly))
+        {
+            CLogMessage(this).error("Failed to open %1: %2") << m_revisionFileName << revisionFile.errorString();
+            return;
+        }
+
+        QUuid newRevision(revisionFile.readAll());
+        if (m_revision != newRevision)
+        {
+            m_revision = newRevision;
+            CVariantMap newValues;
+            loadFromFiles(persistentStore(), newValues);
+            m_deferredChanges.insert(newValues);
+        }
+
+        if (! (m_deferredChanges.isEmpty() || defer))
+        {
+            changeValuesFromRemote(m_deferredChanges, CIdentifier::anonymous());
+            m_deferredChanges.clear();
+        }
+    }
+
+}

src/blackcore/datacache.h

@@ -0,0 +1,110 @@
+/* Copyright (C) 2015
+ * swift project Community / Contributors
+ *
+ * This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
+ * directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
+ * including this file, may be copied, modified, propagated, or distributed except according to the terms
+ * contained in the LICENSE file.
+ */
+
+//! \file
+
+#ifndef BLACKCORE_DATACACHE_H
+#define BLACKCORE_DATACACHE_H
+
+#include "blackcore/blackcoreexport.h"
+#include "blackmisc/valuecache.h"
+
+namespace BlackCore
+{
+
+    /*!
+     * Singleton derived class of CValueCache, for core dynamic data.
+     *
+     * File-based distribution between processes is built-in to the class.
+     */
+    class BLACKCORE_EXPORT CDataCache : public BlackMisc::CValueCache
+    {
+    public:
+        //! Return the singleton instance.
+        static CDataCache *instance();
+
+        //! The directory where core data are stored.
+        static const QString &persistentStore();
+
+    private:
+        CDataCache();
+
+        //! Save values to persistent store. Called whenever a value is changed locally.
+        void saveToStore(const BlackMisc::CVariantMap &values);
+
+        //! Load values from persistent store. Called once per second.
+        //! Also called by saveToStore, to ensure that remote changes to unrelated values are not lost.
+        //! \param lock Whether to acquire the revision file lock. Used when called by saveToStore.
+        //! \param defer Whether to defer applying the changes. Used when called by saveToStore.
+        void loadFromStore(bool lock = true, bool defer = false);
+
+        QTimer m_reloadTimer;
+        QUuid m_revision;
+        const QString m_revisionFileName { persistentStore() + "/.rev" };
+        BlackMisc::CVariantMap m_deferredChanges;
+    };
+
+    /*!
+     * Class template for accessing a specific value in the CDataCache.
+     * \tparam Trait A subclass of BlackCore::CDataTrait that identifies the value's key and other metadata.
+     */
+    template <typename Trait>
+    class CData : public BlackMisc::CCached<typename Trait::type>
+    {
+    public:
+        //! \copydoc BlackMisc::CCached::NotifySlot
+        template <typename T>
+        using NotifySlot = typename BlackMisc::CCached<typename Trait::type>::template NotifySlot<T>;
+
+        //! Constructor.
+        //! \param owner Will be the parent of the internal QObject used to access the value.
+        //! \param slot Slot to call when the value is modified by another object.
+        //!             Must be a void, non-const member function of the owner.
+        template <typename T>
+        CData(T *owner, NotifySlot<T> slot = nullptr) :
+            CData::CCached(CDataCache::instance(), Trait::key(), Trait::isValid, Trait::defaultValue(), owner, slot)
+        {}
+
+        //! Reset the data to its default value.
+        void setDefault() { this->set(Trait::defaultValue()); }
+    };
+
+    /*!
+     * Base class for traits to be used as template argument to BlackCore::CData.
+     */
+    template <typename T>
+    struct CDataTrait
+    {
+        //! Data type of the value.
+        using type = T;
+
+        //! Key string of the value. Reimplemented in derived class.
+        static const char *key() { qFatal("Not implemented"); return ""; }
+
+        //! Validator function. Return true if the argument is valid, false otherwise. Default
+        //! implementation just returns true. Reimplemented in derived class to support validation of the value.
+        static bool isValid(const T &) { return true; }
+
+        //! Return the value to use in case the supplied value does not satisfy the validator.
+        //! Default implementation returns a default-constructed value.
+        static const T &defaultValue() { static const T def {}; return def; }
+
+        //! Deleted default constructor.
+        CDataTrait() = delete;
+
+        //! Deleted copy constructor.
+        CDataTrait(const CDataTrait &) = delete;
+
+        //! Deleted copy assignment operator.
+        CDataTrait &operator =(const CDataTrait &) = delete;
+    };
+
+}
+
+#endif

src/blackmisc/dictionary.h

@@ -220,6 +220,17 @@ namespace BlackMisc
             removeByValueIf(BlackMisc::Predicates::MemberEqual(membFunc, returnValue));
         }
 
+        //! Remove elements for which the same key/value pair is present in an other dictionary.
+        void removeDuplicates(const CDictionary &other)
+        {
+            for (auto it = begin(); it != end();)
+            {
+                auto it2 = other.find(it.key());
+                if (it2 != other.end() && it.value() == it2.value()) { it = erase(it); }
+                else { ++it; }
+            }
+        }
+
         //! \copydoc CValueObject::toJson
         QJsonObject toJson() const
         {

src/blackmisc/identifier.cpp

@@ -35,6 +35,17 @@ namespace BlackMisc
         return id;
     }
 
+    QUuid CIdentifier::toUuid() const
+    {
+        static const QUuid ns = QUuid::createUuid();
+        QByteArray baseData;
+        baseData.append(getMachineId());
+        baseData.append(reinterpret_cast<const char *>(&m_processId), sizeof(m_processId));
+        baseData.append(reinterpret_cast<const char *>(&m_timestampMSecsSinceEpoch), sizeof(m_timestampMSecsSinceEpoch));
+        baseData.append(getName());
+        return QUuid::createUuidV5(ns, baseData);
+    }
+
     QByteArray CIdentifier::getMachineId() const
     {
         return QByteArray::fromBase64(m_machineIdBase64.toLocal8Bit());

src/blackmisc/identifier.h

@@ -49,6 +49,9 @@ namespace BlackMisc
         //! Returns an anonymous identifier.
         static CIdentifier anonymous();
 
+        //! Produces a UUID generated from the identifier.
+        QUuid toUuid() const;
+
         //! Name
         QString getName() const { return m_name; }
 

src/blackmisc/valuecache.cpp

@@ -146,7 +146,14 @@ namespace BlackMisc
 
     CStatusMessage CValueCache::saveToFiles(const QString &dir, const QString &keyPrefix) const
     {
+        QMutexLocker lock(&m_mutex);
         auto values = getAllValues(keyPrefix);
+        return saveToFiles(dir, values);
+    }
+
+    CStatusMessage CValueCache::saveToFiles(const QString &dir, const CVariantMap &values) const
+    {
+        QMutexLocker lock(&m_mutex);
         QMap<QString, CVariantMap> namespaces;
         for (auto it = values.cbegin(); it != values.cend(); ++it)
         {
@@ -182,10 +189,21 @@ namespace BlackMisc
 
     CStatusMessage CValueCache::loadFromFiles(const QString &dir)
     {
+        QMutexLocker lock(&m_mutex);
+        CVariantMap values;
+        auto status = loadFromFiles(dir, values);
+        insertValues(values);
+        return status;
+    }
+
+    CStatusMessage CValueCache::loadFromFiles(const QString &dir, CVariantMap &o_values) const
+    {
+        QMutexLocker lock(&m_mutex);
         if (! QDir(dir).isReadable())
         {
             return CLogMessage(this).error("Failed to read directory %1") << dir;
         }
+        auto currentValues = getAllValues();
         for (const auto &filename : QDir(dir).entryList({ "*.json" }, QDir::Files))
         {
             QFile file(dir + "/" + filename);
@@ -198,7 +216,10 @@ namespace BlackMisc
             {
                 return CLogMessage(this).error("Invalid JSON format in %1") << file.fileName();
             }
-            loadFromJson(json.object());
+            CVariantMap temp;
+            temp.convertFromJson(json.object());
+            temp.removeDuplicates(currentValues);
+            o_values.insert(temp);
         }
         return {};
     }

src/blackmisc/valuecache.h

@@ -91,13 +91,22 @@ namespace BlackMisc
         //! of CValueCache instances in all processes including this one. The slot will do its own round-trip detection.
         void valuesChangedByLocal(const BlackMisc::CVariantMap &values);
 
+    protected:
+        //! Save specific values to Json files in a given directory.
+        CStatusMessage saveToFiles(const QString &directory, const CVariantMap &values) const;
+
+        //! Load from Json files in a given directory any values which differ from the current ones, and insert them in o_values.
+        CStatusMessage loadFromFiles(const QString &directory, CVariantMap &o_values) const;
+
+        //! Mutex protecting operations which are critical on m_elements.
+        mutable QMutex m_mutex { QMutex::Recursive };
+
     private:
         friend class Private::CValuePage;
         struct Element;
         using ElementPtr = QSharedPointer<Element>; // QMap doesn't support move-only types
 
         QMap<QString, ElementPtr> m_elements;
-        mutable QMutex m_mutex { QMutex::Recursive };
 
         Element &getElement(const QString &key);
         Element &getElement(const QString &key, QMap<QString, ElementPtr>::const_iterator pos);