From 70224bea723daa12354e5cca21629d5de51f2cf7 Mon Sep 17 00:00:00 2001 From: Mathew Sutcliffe Date: Thu, 9 Mar 2017 23:23:49 +0000 Subject: [PATCH] refs #904 Settings JSON files split into arbitrary levels of subfolders, one file per key. --- src/blackmisc/datacache.cpp | 4 ++-- src/blackmisc/settingscache.cpp | 4 ++-- src/blackmisc/valuecache.cpp | 25 ++++++++++++++++--------- src/blackmisc/valuecache.h | 5 +++-- tests/blackmisc/testvaluecache.cpp | 18 +++++++++--------- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/blackmisc/datacache.cpp b/src/blackmisc/datacache.cpp index 08f986e42..3c863d911 100644 --- a/src/blackmisc/datacache.cpp +++ b/src/blackmisc/datacache.cpp @@ -69,7 +69,7 @@ namespace BlackMisc CDataCacheRevision *m_rev = nullptr; }; - CDataCache::CDataCache() + CDataCache::CDataCache() : CValueCache(1) { if (! QDir::root().mkpath(persistentStore())) { @@ -109,7 +109,7 @@ namespace BlackMisc QString CDataCache::filenameForKey(const QString &key) { - return CFileUtils::appendFilePaths(persistentStore(), CValueCache::filenameForKey(key)); + return CFileUtils::appendFilePaths(persistentStore(), instance()->CValueCache::filenameForKey(key)); } QStringList CDataCache::enumerateStore() const diff --git a/src/blackmisc/settingscache.cpp b/src/blackmisc/settingscache.cpp index 41504b86f..9c7f9f46b 100644 --- a/src/blackmisc/settingscache.cpp +++ b/src/blackmisc/settingscache.cpp @@ -15,7 +15,7 @@ namespace BlackMisc { - CSettingsCache::CSettingsCache() + CSettingsCache::CSettingsCache() : CValueCache(0) {} CSettingsCache *CSettingsCache::instance() @@ -58,7 +58,7 @@ namespace BlackMisc QString CSettingsCache::filenameForKey(const QString &key) { - return CFileUtils::appendFilePaths(persistentStore(), CValueCache::filenameForKey(key)); + return CFileUtils::appendFilePaths(persistentStore(), instance()->CValueCache::filenameForKey(key)); } const QString CSettingsCache::relativeFilePath() diff --git a/src/blackmisc/valuecache.cpp b/src/blackmisc/valuecache.cpp index d4e1e3c4e..281743fa5 100644 --- a/src/blackmisc/valuecache.cpp +++ b/src/blackmisc/valuecache.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -162,8 +163,9 @@ namespace BlackMisc return cats; } - CValueCache::CValueCache(QObject *parent) : QObject(parent) + CValueCache::CValueCache(int fileSplitDepth, QObject *parent) : QObject(parent), m_fileSplitDepth(fileSplitDepth) { + Q_ASSERT_X(fileSplitDepth >= 0, Q_FUNC_INFO, "Negative value not allowed, use 0 for maximum split depth"); Q_ASSERT_X(QThread::currentThread() == qApp->thread(), Q_FUNC_INFO, "Cache constructed in wrong thread"); } @@ -365,7 +367,7 @@ namespace BlackMisc QMap namespaces; for (auto it = values.cbegin(); it != values.cend(); ++it) { - namespaces[it.key().section('/', 0, 0)].insert(it.key(), it.value()); + namespaces[it.key().section('/', 0, m_fileSplitDepth - 1)].insert(it.key(), it.value()); } if (! QDir::root().mkpath(dir)) { @@ -374,6 +376,10 @@ namespace BlackMisc for (auto it = namespaces.cbegin(); it != namespaces.cend(); ++it) { CAtomicFile file(dir + "/" + it.key() + ".json"); + if (! QDir::root().mkpath(QFileInfo(file).path())) + { + return CStatusMessage(this).error("Failed to create directory '%1'") << QFileInfo(file).path(); + } if (! file.open(QFile::ReadWrite | QFile::Text)) { return CStatusMessage(this).error("Failed to open %1: %2") << file.fileName() << file.errorString(); @@ -419,19 +425,20 @@ namespace BlackMisc QMap keysInFiles; for (const auto &key : keys) { - keysInFiles[key.section('/', 0, 0)].push_back(key); + keysInFiles[key.section('/', 0, m_fileSplitDepth - 1) + ".json"].push_back(key); } if (keys.isEmpty()) { - for (const auto &filename : QDir(dir).entryInfoList({ "*.json" }, QDir::Files)) + QDirIterator iter(dir, { "*.json" }, QDir::Files, QDirIterator::Subdirectories); + while (iter.hasNext()) { - keysInFiles.insert(filename.completeBaseName(), {}); + keysInFiles.insert(QDir(dir).relativeFilePath(iter.next()), {}); } } bool ok = true; for (auto it = keysInFiles.cbegin(); it != keysInFiles.cend(); ++it) { - QFile file(dir + "/" + it.key() + ".json"); + QFile file(QDir(dir).absoluteFilePath(it.key())); if (! file.exists()) { continue; @@ -453,7 +460,7 @@ namespace BlackMisc } else { - const QString messagePrefix = QStringLiteral("Parsing %1.json").arg(it.key()); + const QString messagePrefix = QStringLiteral("Parsing %1").arg(it.key()); auto messages = temp.convertFromMemoizedJsonNoThrow(json.object(), it.value(), this, messagePrefix); if (it.value().isEmpty()) { messages.push_back(temp.convertFromMemoizedJsonNoThrow(json.object(), this, messagePrefix)); } if (! messages.isEmpty()) @@ -508,9 +515,9 @@ namespace BlackMisc } } - QString CValueCache::filenameForKey(const QString &key) + QString CValueCache::filenameForKey(const QString &key) const { - return key.section('/', 0, 0) + ".json"; + return key.section('/', 0, m_fileSplitDepth - 1) + ".json"; } QStringList CValueCache::enumerateFiles(const QString &dir) const diff --git a/src/blackmisc/valuecache.h b/src/blackmisc/valuecache.h index 67090056a..92d394aa7 100644 --- a/src/blackmisc/valuecache.h +++ b/src/blackmisc/valuecache.h @@ -163,7 +163,7 @@ namespace BlackMisc static const CLogCategoryList &getLogCategories(); //! Constructor. - explicit CValueCache(QObject *parent = nullptr); + explicit CValueCache(int fileSplitDepth, QObject *parent = nullptr); //! Return map containing all values in the cache. //! If prefix is provided then only those values whose keys start with that prefix. @@ -219,7 +219,7 @@ namespace BlackMisc //! Return the (relative) filename that may is (or would be) used to save the value with the given key. //! The file may or may not exist (because it might not have been saved yet). //! \threadsafe - static QString filenameForKey(const QString &key); + QString filenameForKey(const QString &key) const; //! List the Json files which are (or would be) used to save the current values. //! The files may or may not exist (because they might not have been saved yet). @@ -317,6 +317,7 @@ namespace BlackMisc QMap m_elements; QMap m_humanReadable; + const int m_fileSplitDepth = 1; //!< How many levels of subdirectories to split JSON files Element &getElement(const QString &key); Element &getElement(const QString &key, QMap::const_iterator pos); diff --git a/tests/blackmisc/testvaluecache.cpp b/tests/blackmisc/testvaluecache.cpp index 1925d7cd4..3ce9228b6 100644 --- a/tests/blackmisc/testvaluecache.cpp +++ b/tests/blackmisc/testvaluecache.cpp @@ -77,7 +77,7 @@ namespace BlackMiscTest { "value4", CVariant::from(4) } }; - CValueCache cache; + CValueCache cache(1); QVERIFY(cache.getAllValues() == CVariantMap()); cache.insertValues({ testData, QDateTime::currentMSecsSinceEpoch() }); QVERIFY(cache.getAllValues() == testData); @@ -131,7 +131,7 @@ namespace BlackMiscTest void CTestValueCache::localOnly() { - CValueCache cache; + CValueCache cache(1); for (int i = 0; i < 2; ++i) { QTest::ignoreMessage(QtDebugMsg, QRegularExpression("Empty cache value")); } CValueCacheUser user1(&cache); CValueCacheUser user2(&cache); @@ -140,7 +140,7 @@ namespace BlackMiscTest void CTestValueCache::localOnlyWithThreads() { - CValueCache cache; + CValueCache cache(1); for (int i = 0; i < 2; ++i) { QTest::ignoreMessage(QtDebugMsg, QRegularExpression("Empty cache value")); } CValueCacheUser user1(&cache); CValueCacheUser user2(&cache); @@ -158,8 +158,8 @@ namespace BlackMiscTest json.insert("processId", otherProcess.getProcessId() + 1); otherProcess.convertFromJson(json); - CValueCache thisCache; - CValueCache otherCache; + CValueCache thisCache(1); + CValueCache otherCache(1); connect(&thisCache, &CValueCache::valuesChangedByLocal, &thisCache, [ & ](const CValueCachePacket &values) { QMetaObject::invokeMethod(&thisCache, "changeValuesFromRemote", Q_ARG(BlackMisc::CValueCachePacket, values), Q_ARG(BlackMisc::CIdentifier, thisProcess)); @@ -191,7 +191,7 @@ namespace BlackMiscTest void CTestValueCache::batched() { - CValueCache cache; + CValueCache cache(1); for (int i = 0; i < 2; ++i) { QTest::ignoreMessage(QtDebugMsg, QRegularExpression("Empty cache value")); } CValueCacheUser user1(&cache); CValueCacheUser user2(&cache); @@ -225,7 +225,7 @@ namespace BlackMiscTest { "value3", CVariant::from(3) } }; - CValueCache cache; + CValueCache cache(1); cache.loadFromJson(testJson); QVERIFY(cache.getAllValues() == testData); QVERIFY(cache.saveToJson() == testJson); @@ -243,7 +243,7 @@ namespace BlackMiscTest { "namespace2/aircraft", CVariant::from(aircraft) }, { "namespace2/atcstations", CVariant::from(atcStations) } }; - CValueCache cache; + CValueCache cache(1); cache.insertValues({ testData, QDateTime::currentMSecsSinceEpoch() }); QDir dir(QDir::currentPath() + "/testcache"); @@ -257,7 +257,7 @@ namespace BlackMiscTest QCOMPARE(files[0].fileName(), QString("namespace1.json")); QCOMPARE(files[1].fileName(), QString("namespace2.json")); - CValueCache cache2; + CValueCache cache2(1); status = cache2.loadFromFiles(dir.absolutePath()); QVERIFY(status.isSuccess()); QCOMPARE(cache2.getAllValues(), testData);