Ref T237, init/read from JSON files

* multi format compatible (automatically detect format)
* flag if cache values shall be overridden
* init from resource files (those are the files coming with the installer)
* automatically read in background if reader is (already) in its own thread (otherwise cache.set() ASSERT)
This commit is contained in:
Klaus Basan
2018-01-30 06:34:17 +01:00
parent 28b5ec9ec6
commit f3afde76ff
14 changed files with 369 additions and 215 deletions

View File

@@ -15,6 +15,7 @@
#include "blackcore/application.h"
#include "blackmisc/db/datastoreutility.h"
#include "blackmisc/network/networkutils.h"
#include "blackmisc/network/entityflags.h"
#include "blackmisc/directoryutils.h"
#include "blackmisc/logcategory.h"
#include "blackmisc/logcategorylist.h"
@@ -23,6 +24,7 @@
#include <QByteArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QFileInfo>
#include <QJsonValueRef>
#include <QMetaObject>
#include <QNetworkReply>
@@ -568,6 +570,11 @@ namespace BlackCore
return m_1stReplyReceived;
}
QString CDatabaseReader::getSupportedEntitiesAsString() const
{
return CEntityFlags::flagToString(this->getSupportedEntities());
}
CEntityFlags::Entity CDatabaseReader::maskBySupportedEntities(CEntityFlags::Entity entities) const
{
return entities & this->getSupportedEntities();
@@ -590,27 +597,20 @@ namespace BlackCore
return m_statusMessage;
}
CStatusMessageList CDatabaseReader::initFromLocalResourceFiles()
CStatusMessageList CDatabaseReader::initFromLocalResourceFiles(bool inBackground)
{
static const QDateTime threshold = QDateTime::currentDateTimeUtc().addYears(-1);
const QSet<CEntityFlags::Entity> eSet = CEntityFlags::asSingleEntities(this->getSupportedEntities());
CStatusMessageList msgs;
CEntityFlags::Entity entities = CEntityFlags::NoEntity;
for (CEntityFlags::Entity e : eSet)
const bool overrideNewerOnly = true;
if (inBackground || !CThreadUtils::isCurrentThreadObjectThread(this))
{
if (this->hasCacheTimestampNewerThan(e, threshold))
{
entities |= e;
}
{
msgs.push_back(CStatusMessage(this).info("Will not init from local file, there are already data for '%1'") << CEntityFlags::flagToString(e));
}
const bool s = this->readFromJsonFilesInBackground(CDirectoryUtils::staticDbFilesDirectory(), this->getSupportedEntities(), overrideNewerOnly);
return s ?
CStatusMessage(this).info("Started reading in background from '%1' of entities: '%2'") << CDirectoryUtils::staticDbFilesDirectory() << CEntityFlags::flagToString(this->getSupportedEntities()) :
CStatusMessage(this).error("Starting reading in background from '%1' of entities: '%2' failed") << CDirectoryUtils::staticDbFilesDirectory() << CEntityFlags::flagToString(this->getSupportedEntities());
}
else
{
return this->readFromJsonFiles(CDirectoryUtils::staticDbFilesDirectory(), this->getSupportedEntities(), overrideNewerOnly);
}
const CStatusMessageList readMsgs = this->readFromJsonFiles(CDirectoryUtils::staticDbFilesDirectory(), entities);
msgs.push_back(readMsgs);
return readMsgs;
}
void CDatabaseReader::setReplyStatus(QNetworkReply::NetworkError status, const QString &message)
@@ -631,6 +631,27 @@ namespace BlackCore
}
}
bool CDatabaseReader::overrideCacheFromFile(bool overrideNewerOnly, const QFileInfo &fileInfo, CEntityFlags::Entity entity, CStatusMessageList &msgs) const
{
if (!fileInfo.created().isValid()) { return false; }
if (!overrideNewerOnly) { return true; }
const qint64 fileTs = fileInfo.created().toUTC().toMSecsSinceEpoch();
const QDateTime cacheDateTime(this->getCacheTimestamp(entity));
if (!cacheDateTime.isValid()) { return true; } // no cache
const qint64 cacheTs = cacheDateTime.toUTC().toMSecsSinceEpoch();
if (fileTs > cacheTs)
{
msgs.push_back(CStatusMessage(this).info("File '%1' is newer than cache (%2)") << fileInfo.absoluteFilePath() << cacheDateTime.toUTC().toString());
return true;
}
else
{
msgs.push_back(CStatusMessage(this).info("File '%1' is not newer than cache (%2)") << fileInfo.absoluteFilePath() << cacheDateTime.toUTC().toString());
return true;
}
}
QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode)
{
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity");