refs #904 Settings JSON files split into arbitrary levels of subfolders, one file per key.

src/blackmisc/datacache.cpp

@@ -69,7 +69,7 @@ namespace BlackMisc
         CDataCacheRevision *m_rev = nullptr;
     };
 
-    CDataCache::CDataCache()
+    CDataCache::CDataCache() : CValueCache(1)
     {
         if (! QDir::root().mkpath(persistentStore()))
         {
@@ -109,7 +109,7 @@ namespace BlackMisc
 
     QString CDataCache::filenameForKey(const QString &key)
     {
-        return CFileUtils::appendFilePaths(persistentStore(), CValueCache::filenameForKey(key));
+        return CFileUtils::appendFilePaths(persistentStore(), instance()->CValueCache::filenameForKey(key));
     }
 
     QStringList CDataCache::enumerateStore() const

src/blackmisc/settingscache.cpp

@@ -15,7 +15,7 @@
 
 namespace BlackMisc
 {
-    CSettingsCache::CSettingsCache()
+    CSettingsCache::CSettingsCache() : CValueCache(0)
     {}
 
     CSettingsCache *CSettingsCache::instance()
@@ -58,7 +58,7 @@ namespace BlackMisc
 
     QString CSettingsCache::filenameForKey(const QString &key)
     {
-        return CFileUtils::appendFilePaths(persistentStore(), CValueCache::filenameForKey(key));
+        return CFileUtils::appendFilePaths(persistentStore(), instance()->CValueCache::filenameForKey(key));
     }
 
     const QString CSettingsCache::relativeFilePath()

src/blackmisc/valuecache.cpp

@@ -22,6 +22,7 @@
 #include <QCoreApplication>
 #include <QDBusMetaType>
 #include <QDir>
+#include <QDirIterator>
 #include <QFileInfo>
 #include <QFlags>
 #include <QIODevice>
@@ -162,8 +163,9 @@ namespace BlackMisc
         return cats;
     }
 
-    CValueCache::CValueCache(QObject *parent) : QObject(parent)
+    CValueCache::CValueCache(int fileSplitDepth, QObject *parent) : QObject(parent), m_fileSplitDepth(fileSplitDepth)
     {
+        Q_ASSERT_X(fileSplitDepth >= 0, Q_FUNC_INFO, "Negative value not allowed, use 0 for maximum split depth");
         Q_ASSERT_X(QThread::currentThread() == qApp->thread(), Q_FUNC_INFO, "Cache constructed in wrong thread");
     }
 
@@ -365,7 +367,7 @@ namespace BlackMisc
         QMap<QString, CVariantMap> namespaces;
         for (auto it = values.cbegin(); it != values.cend(); ++it)
         {
-            namespaces[it.key().section('/', 0, 0)].insert(it.key(), it.value());
+            namespaces[it.key().section('/', 0, m_fileSplitDepth - 1)].insert(it.key(), it.value());
         }
         if (! QDir::root().mkpath(dir))
         {
@@ -374,6 +376,10 @@ namespace BlackMisc
         for (auto it = namespaces.cbegin(); it != namespaces.cend(); ++it)
         {
             CAtomicFile file(dir + "/" + it.key() + ".json");
+            if (! QDir::root().mkpath(QFileInfo(file).path()))
+            {
+                return CStatusMessage(this).error("Failed to create directory '%1'") << QFileInfo(file).path();
+            }
             if (! file.open(QFile::ReadWrite | QFile::Text))
             {
                 return CStatusMessage(this).error("Failed to open %1: %2") << file.fileName() << file.errorString();
@@ -419,19 +425,20 @@ namespace BlackMisc
         QMap<QString, QStringList> keysInFiles;
         for (const auto &key : keys)
         {
-            keysInFiles[key.section('/', 0, 0)].push_back(key);
+            keysInFiles[key.section('/', 0, m_fileSplitDepth - 1) + ".json"].push_back(key);
         }
         if (keys.isEmpty())
         {
-            for (const auto &filename : QDir(dir).entryInfoList({ "*.json" }, QDir::Files))
+            QDirIterator iter(dir, { "*.json" }, QDir::Files, QDirIterator::Subdirectories);
+            while (iter.hasNext())
             {
-                keysInFiles.insert(filename.completeBaseName(), {});
+                keysInFiles.insert(QDir(dir).relativeFilePath(iter.next()), {});
             }
         }
         bool ok = true;
         for (auto it = keysInFiles.cbegin(); it != keysInFiles.cend(); ++it)
         {
-            QFile file(dir + "/" + it.key() + ".json");
+            QFile file(QDir(dir).absoluteFilePath(it.key()));
             if (! file.exists())
             {
                 continue;
@@ -453,7 +460,7 @@ namespace BlackMisc
             }
             else
             {
-                const QString messagePrefix = QStringLiteral("Parsing %1.json").arg(it.key());
+                const QString messagePrefix = QStringLiteral("Parsing %1").arg(it.key());
                 auto messages = temp.convertFromMemoizedJsonNoThrow(json.object(), it.value(), this, messagePrefix);
                 if (it.value().isEmpty()) { messages.push_back(temp.convertFromMemoizedJsonNoThrow(json.object(), this, messagePrefix)); }
                 if (! messages.isEmpty())
@@ -508,9 +515,9 @@ namespace BlackMisc
         }
     }
 
-    QString CValueCache::filenameForKey(const QString &key)
+    QString CValueCache::filenameForKey(const QString &key) const
     {
-        return key.section('/', 0, 0) + ".json";
+        return key.section('/', 0, m_fileSplitDepth - 1) + ".json";
     }
 
     QStringList CValueCache::enumerateFiles(const QString &dir) const

src/blackmisc/valuecache.h

@@ -163,7 +163,7 @@ namespace BlackMisc
         static const CLogCategoryList &getLogCategories();
 
         //! Constructor.
-        explicit CValueCache(QObject *parent = nullptr);
+        explicit CValueCache(int fileSplitDepth, QObject *parent = nullptr);
 
         //! Return map containing all values in the cache.
         //! If prefix is provided then only those values whose keys start with that prefix.
@@ -219,7 +219,7 @@ namespace BlackMisc
         //! Return the (relative) filename that may is (or would be) used to save the value with the given key.
         //! The file may or may not exist (because it might not have been saved yet).
         //! \threadsafe
-        static QString filenameForKey(const QString &key);
+        QString filenameForKey(const QString &key) const;
 
         //! List the Json files which are (or would be) used to save the current values.
         //! The files may or may not exist (because they might not have been saved yet).
@@ -317,6 +317,7 @@ namespace BlackMisc
 
         QMap<QString, ElementPtr> m_elements;
         QMap<QString, QString> m_humanReadable;
+        const int m_fileSplitDepth = 1; //!< How many levels of subdirectories to split JSON files
 
         Element &getElement(const QString &key);
         Element &getElement(const QString &key, QMap<QString, ElementPtr>::const_iterator pos);

tests/blackmisc/testvaluecache.cpp

@@ -77,7 +77,7 @@ namespace BlackMiscTest
             { "value4", CVariant::from(4) }
         };
 
-        CValueCache cache;
+        CValueCache cache(1);
         QVERIFY(cache.getAllValues() == CVariantMap());
         cache.insertValues({ testData, QDateTime::currentMSecsSinceEpoch() });
         QVERIFY(cache.getAllValues() == testData);
@@ -131,7 +131,7 @@ namespace BlackMiscTest
 
     void CTestValueCache::localOnly()
     {
-        CValueCache cache;
+        CValueCache cache(1);
         for (int i = 0; i < 2; ++i) { QTest::ignoreMessage(QtDebugMsg, QRegularExpression("Empty cache value")); }
         CValueCacheUser user1(&cache);
         CValueCacheUser user2(&cache);
@@ -140,7 +140,7 @@ namespace BlackMiscTest
 
     void CTestValueCache::localOnlyWithThreads()
     {
-        CValueCache cache;
+        CValueCache cache(1);
         for (int i = 0; i < 2; ++i) { QTest::ignoreMessage(QtDebugMsg, QRegularExpression("Empty cache value")); }
         CValueCacheUser user1(&cache);
         CValueCacheUser user2(&cache);
@@ -158,8 +158,8 @@ namespace BlackMiscTest
         json.insert("processId", otherProcess.getProcessId() + 1);
         otherProcess.convertFromJson(json);
 
-        CValueCache thisCache;
+        CValueCache thisCache(1);
-        CValueCache otherCache;
+        CValueCache otherCache(1);
         connect(&thisCache, &CValueCache::valuesChangedByLocal, &thisCache, [ & ](const CValueCachePacket &values)
         {
             QMetaObject::invokeMethod(&thisCache, "changeValuesFromRemote", Q_ARG(BlackMisc::CValueCachePacket, values), Q_ARG(BlackMisc::CIdentifier, thisProcess));
@@ -191,7 +191,7 @@ namespace BlackMiscTest
 
     void CTestValueCache::batched()
     {
-        CValueCache cache;
+        CValueCache cache(1);
         for (int i = 0; i < 2; ++i) { QTest::ignoreMessage(QtDebugMsg, QRegularExpression("Empty cache value")); }
         CValueCacheUser user1(&cache);
         CValueCacheUser user2(&cache);
@@ -225,7 +225,7 @@ namespace BlackMiscTest
             { "value3", CVariant::from(3) }
         };
 
-        CValueCache cache;
+        CValueCache cache(1);
         cache.loadFromJson(testJson);
         QVERIFY(cache.getAllValues() == testData);
         QVERIFY(cache.saveToJson() == testJson);
@@ -243,7 +243,7 @@ namespace BlackMiscTest
             { "namespace2/aircraft", CVariant::from(aircraft) },
             { "namespace2/atcstations", CVariant::from(atcStations) }
         };
-        CValueCache cache;
+        CValueCache cache(1);
         cache.insertValues({ testData, QDateTime::currentMSecsSinceEpoch() });
 
         QDir dir(QDir::currentPath() + "/testcache");
@@ -257,7 +257,7 @@ namespace BlackMiscTest
         QCOMPARE(files[0].fileName(), QString("namespace1.json"));
         QCOMPARE(files[1].fileName(), QString("namespace2.json"));
 
-        CValueCache cache2;
+        CValueCache cache2(1);
         status = cache2.loadFromFiles(dir.absolutePath());
         QVERIFY(status.isSuccess());
         QCOMPARE(cache2.getAllValues(), testData);