mirror of
https://github.com/swift-project/pilotclient.git
synced 2026-03-23 07:15:35 +08:00
f3847bd33c
* in case the internet provider is down, this is not detected * extended test to detect if access is possible * in case of downtime this indicated in status component
709 lines
32 KiB
C++
709 lines
32 KiB
C++
/* Copyright (C) 2015
|
|
* swift project Community / Contributors
|
|
*
|
|
* This file is part of swift project. It is subject to the license terms in the LICENSE file found in the top-level
|
|
* directory of this distribution and at http://www.swift-project.org/license.html. No part of swift project,
|
|
* including this file, may be copied, modified, propagated, or distributed except according to the terms
|
|
* contained in the LICENSE file.
|
|
*/
|
|
|
|
#include "blackcore/db/databasereader.h"
|
|
#include "blackcore/db/infodatareader.h"
|
|
#include "blackcore/db/databaseutils.h"
|
|
#include "blackcore/webdataservices.h"
|
|
#include "blackcore/application.h"
|
|
#include "blackmisc/db/datastoreutility.h"
|
|
#include "blackmisc/network/networkutils.h"
|
|
#include "blackmisc/logcategory.h"
|
|
#include "blackmisc/logcategorylist.h"
|
|
#include "blackmisc/logmessage.h"
|
|
|
|
#include <QByteArray>
|
|
#include <QJsonDocument>
|
|
#include <QJsonObject>
|
|
#include <QJsonValueRef>
|
|
#include <QMetaObject>
|
|
#include <QNetworkReply>
|
|
#include <QReadLocker>
|
|
#include <QUrl>
|
|
#include <QWriteLocker>
|
|
|
|
using namespace BlackMisc;
|
|
using namespace BlackMisc::Db;
|
|
using namespace BlackMisc::Network;
|
|
using namespace BlackCore;
|
|
using namespace BlackCore::Data;
|
|
using namespace BlackCore::Db;
|
|
|
|
namespace BlackCore
|
|
{
|
|
namespace Db
|
|
{
|
|
CUrl CDatabaseReader::s_workingSharedDbData;
|
|
|
|
CDatabaseReader::CDatabaseReader(QObject *owner, const CDatabaseReaderConfigList &config, const QString &name) :
|
|
BlackCore::CThreadedReader(owner, name), m_config(config)
|
|
{
|
|
CDatabaseReader::initWorkingUrls();
|
|
}
|
|
|
|
void CDatabaseReader::readInBackgroundThread(CEntityFlags::Entity entities, const QDateTime &newerThan)
|
|
{
|
|
if (!this->doWorkCheck()) { return; }
|
|
|
|
// we accept cached data
|
|
Q_ASSERT_X(!entities.testFlag(CEntityFlags::DbInfoObjectEntity), Q_FUNC_INFO, "Read info objects directly");
|
|
const bool hasInfoObjects = this->hasDbInfoObjects(); // no info objects is not necessarily an error, but indicates a) either data not available (DB down) or b) only caches are used
|
|
CEntityFlags::Entity allEntities = entities;
|
|
CEntityFlags::Entity cachedEntities = CEntityFlags::NoEntity;
|
|
CEntityFlags::Entity dbEntities = CEntityFlags::NoEntity;
|
|
CEntityFlags::Entity sharedEntities = CEntityFlags::NoEntity;
|
|
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); // CEntityFlags::InfoObjectEntity will be ignored
|
|
while (currentEntity)
|
|
{
|
|
const CDatabaseReaderConfig config(this->getConfigForEntity(currentEntity));
|
|
const QString currentEntityName = CEntityFlags::flagToString(currentEntity);
|
|
|
|
// retrieval mode
|
|
const CDbFlags::DataRetrievalMode rm = config.getRetrievalMode(); // DB reading, cache, shared
|
|
Q_ASSERT_X(!rm.testFlag(CDbFlags::Unspecified), Q_FUNC_INFO, "Missing retrieval mode");
|
|
const QString rmString = CDbFlags::flagToString(rm);
|
|
const CDbFlags::DataRetrievalModeFlag rmDbOrSharedFlag = CDbFlags::modeToModeFlag(rm & CDbFlags::DbReadingOrShared);
|
|
const QString rmDbOrSharedFlagString = CDbFlags::flagToString(rmDbOrSharedFlag);
|
|
const bool rmDbReadingOrShared = (rmDbOrSharedFlag == CDbFlags::DbReading || rmDbOrSharedFlag == CDbFlags::Shared);
|
|
|
|
if (rm.testFlag(CDbFlags::Ignore) || rm.testFlag(CDbFlags::Canceled))
|
|
{
|
|
// do not load
|
|
}
|
|
else if (rm.testFlag(CDbFlags::Cached))
|
|
{
|
|
// info object comparisons only for: cache + shared or cache + DB data
|
|
if (hasInfoObjects && rmDbReadingOrShared)
|
|
{
|
|
// check mode here for consistency
|
|
Q_ASSERT_X(!getBaseUrl(rmDbOrSharedFlag).isEmpty(), Q_FUNC_INFO, "Wrong retrieval mode");
|
|
|
|
const bool changedUrl = this->hasChangedUrl(currentEntity);
|
|
const QDateTime cacheTs(this->getCacheTimestamp(currentEntity));
|
|
const QDateTime latestEntityTs(this->getLatestEntityTimestampFromDbInfoObjects(currentEntity));
|
|
const qint64 cacheTimestamp = cacheTs.isValid() ? cacheTs.toMSecsSinceEpoch() : -1;
|
|
const qint64 latestEntityTimestamp = latestEntityTs.isValid() ? latestEntityTs.toMSecsSinceEpoch() : -1;
|
|
Q_ASSERT_X(latestEntityTimestamp >= 0, Q_FUNC_INFO, "Missing timestamp");
|
|
if (!changedUrl && cacheTimestamp >= latestEntityTimestamp && cacheTimestamp >= 0 && latestEntityTimestamp >= 0)
|
|
{
|
|
this->admitCaches(currentEntity);
|
|
cachedEntities |= currentEntity; // read from cache
|
|
CLogMessage(this).info("Using cache for '%1' (%2, %3)") << currentEntityName << cacheTs.toString() << cacheTimestamp;
|
|
}
|
|
else
|
|
{
|
|
Q_ASSERT_X(rmDbReadingOrShared, Q_FUNC_INFO, "Wrong retrieval mode");
|
|
if (rmDbOrSharedFlag == CDbFlags::DbReading) { dbEntities |= currentEntity; }
|
|
else if (rmDbOrSharedFlag == CDbFlags::Shared) { sharedEntities |= currentEntity; }
|
|
|
|
if (changedUrl)
|
|
{
|
|
CLogMessage(this).info("Data location changed, will override cache for '%1' reading '%2'") << currentEntityName << rmDbOrSharedFlagString;
|
|
}
|
|
else
|
|
{
|
|
CLogMessage(this).info("Cache for '%1' outdated, latest entity (%2, %3), reading '%4'") << currentEntityName << latestEntityTs.toString() << latestEntityTimestamp << rmDbOrSharedFlagString;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// no info objects, server down or no shared/db read mode
|
|
this->admitCaches(currentEntity);
|
|
if (!rmDbReadingOrShared)
|
|
{
|
|
CLogMessage(this).info("Triggered reading cache for '%1', read mode: %2") << currentEntityName << rmString;
|
|
}
|
|
else
|
|
{
|
|
CLogMessage(this).info("No info object for '%1', triggered reading cache, read mode: %2") << currentEntityName << rmString;
|
|
}
|
|
cachedEntities |= currentEntity; // read from cache
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// cache ignored
|
|
Q_ASSERT_X(rmDbReadingOrShared, Q_FUNC_INFO, "Wrong retrieval mode");
|
|
if (rmDbOrSharedFlag == CDbFlags::DbReading) { dbEntities |= currentEntity; }
|
|
else if (rmDbOrSharedFlag == CDbFlags::Shared) { sharedEntities |= currentEntity; }
|
|
}
|
|
currentEntity = CEntityFlags::iterateDbEntities(allEntities);
|
|
}
|
|
|
|
// signals for the cached entities
|
|
if (cachedEntities != CEntityFlags::NoEntity)
|
|
{
|
|
this->emitReadSignalPerSingleCachedEntity(cachedEntities, true);
|
|
}
|
|
|
|
// Real read from DB
|
|
if (dbEntities != CEntityFlags::NoEntity)
|
|
{
|
|
CLogMessage(this).info("Start reading DB entities: %1") << CEntityFlags::flagToString(dbEntities);
|
|
this->startReadFromBackendInBackgroundThread(dbEntities, CDbFlags::DbReading, newerThan);
|
|
}
|
|
|
|
// Real read from shared
|
|
if (sharedEntities != CEntityFlags::NoEntity)
|
|
{
|
|
CLogMessage(this).info("Start reading shared entities: %1") << CEntityFlags::flagToString(sharedEntities);
|
|
this->startReadFromBackendInBackgroundThread(dbEntities, CDbFlags::Shared, newerThan);
|
|
}
|
|
}
|
|
|
|
CEntityFlags::Entity CDatabaseReader::triggerLoadingDirectlyFromDb(CEntityFlags::Entity entities, const QDateTime &newerThan)
|
|
{
|
|
this->startReadFromBackendInBackgroundThread(entities, CDbFlags::DbReading, newerThan);
|
|
return entities;
|
|
}
|
|
|
|
CEntityFlags::Entity CDatabaseReader::triggerLoadingDirectlyFromSharedFiles(CEntityFlags::Entity entities, bool checkCacheTsUpfront)
|
|
{
|
|
if (entities == CEntityFlags::NoEntity) { return CEntityFlags::NoEntity; }
|
|
if (checkCacheTsUpfront)
|
|
{
|
|
CEntityFlags::Entity newerHeaderEntities = this->getEntitesWithNewerSharedInfoObject(entities);
|
|
if (newerHeaderEntities != entities)
|
|
{
|
|
const CEntityFlags::Entity validInCacheEntities = (entities ^ newerHeaderEntities) & entities;
|
|
CLogMessage(this).info("Reduced '%1' to '%2' before triggering load of shared files (still in cache)") << CEntityFlags::flagToString(entities) << CEntityFlags::flagToString(newerHeaderEntities);
|
|
|
|
// In case we have difference entities we treat them as they were loaded, they result from still being in the cache
|
|
// Using timer to first finish this function, then the resulting signal
|
|
if (validInCacheEntities != CEntityFlags::NoEntity)
|
|
{
|
|
QTimer::singleShot(0, this, [ = ]
|
|
{
|
|
emit this->dataRead(validInCacheEntities, CEntityFlags::ReadFinished, 0);
|
|
});
|
|
}
|
|
if (newerHeaderEntities == CEntityFlags::NoEntity) { return CEntityFlags::NoEntity; }
|
|
entities = newerHeaderEntities;
|
|
}
|
|
}
|
|
this->startReadFromBackendInBackgroundThread(entities, CDbFlags::Shared);
|
|
return entities;
|
|
}
|
|
|
|
void CDatabaseReader::startReadFromBackendInBackgroundThread(CEntityFlags::Entity entities, CDbFlags::DataRetrievalModeFlag mode, const QDateTime &newerThan)
|
|
{
|
|
Q_ASSERT_X(mode == CDbFlags::DbReading || mode == CDbFlags::Shared, Q_FUNC_INFO, "Wrong mode");
|
|
|
|
// ps_read is implemented in the derived classes
|
|
if (entities == CEntityFlags::NoEntity) { return; }
|
|
if (!this->isInternetAccessible(QString("No network/internet access, will not read %1").arg(CEntityFlags::flagToString(entities)))) { return; }
|
|
|
|
const bool s = QMetaObject::invokeMethod(this, "ps_read",
|
|
Q_ARG(BlackMisc::Network::CEntityFlags::Entity, entities),
|
|
Q_ARG(BlackMisc::Db::CDbFlags::DataRetrievalModeFlag, mode),
|
|
Q_ARG(QDateTime, newerThan));
|
|
Q_ASSERT_X(s, Q_FUNC_INFO, "Invoke failed");
|
|
Q_UNUSED(s);
|
|
}
|
|
|
|
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const
|
|
{
|
|
Q_ASSERT_X(nwReply, Q_FUNC_INFO, "missing reply");
|
|
JsonDatastoreResponse datastoreResponse;
|
|
const bool ok = this->setHeaderInfoPart(datastoreResponse, nwReply);
|
|
if (ok)
|
|
{
|
|
const QString dataFileData = nwReply->readAll();
|
|
nwReply->close(); // close asap
|
|
if (dataFileData.isEmpty())
|
|
{
|
|
datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data"));
|
|
}
|
|
else
|
|
{
|
|
CDatabaseReader::stringToDatastoreResponse(dataFileData, datastoreResponse);
|
|
}
|
|
}
|
|
return datastoreResponse;
|
|
}
|
|
|
|
CDatabaseReader::HeaderResponse CDatabaseReader::transformReplyIntoHeaderResponse(QNetworkReply *nwReply) const
|
|
{
|
|
HeaderResponse headerResponse;
|
|
const bool success = this->setHeaderInfoPart(headerResponse, nwReply);
|
|
Q_UNUSED(success);
|
|
return headerResponse;
|
|
}
|
|
|
|
bool CDatabaseReader::setHeaderInfoPart(CDatabaseReader::HeaderResponse &headerResponse, QNetworkReply *nwReply) const
|
|
{
|
|
Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing reply");
|
|
this->threadAssertCheck();
|
|
if (!this->doWorkCheck())
|
|
{
|
|
nwReply->abort();
|
|
headerResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Terminated data parsing process"));
|
|
return false; // stop, terminate straight away, ending thread
|
|
}
|
|
|
|
headerResponse.setUrl(nwReply->url());
|
|
const QVariant started = nwReply->property("started");
|
|
if (started.isValid() && started.canConvert<qint64>())
|
|
{
|
|
const qint64 now = QDateTime::currentMSecsSinceEpoch();
|
|
const qint64 start = started.value<qint64>();
|
|
headerResponse.setLoadTimeMs(now - start);
|
|
}
|
|
|
|
const QDateTime lastModified = nwReply->header(QNetworkRequest::LastModifiedHeader).toDateTime();
|
|
const qulonglong size = nwReply->header(QNetworkRequest::ContentLengthHeader).toULongLong();
|
|
headerResponse.setLastModifiedTimestamp(lastModified);
|
|
headerResponse.setContentLengthHeader(size);
|
|
|
|
if (nwReply->error() == QNetworkReply::NoError)
|
|
{
|
|
// do not close because of obtaining data
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
// no valid response
|
|
const QString error(nwReply->errorString());
|
|
const QString url(nwReply->url().toString());
|
|
nwReply->abort();
|
|
headerResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError,
|
|
QString("Reading data failed: '" + error + "' " + url)));
|
|
return false;
|
|
}
|
|
}
|
|
|
|
CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply)
|
|
{
|
|
this->setReplyStatus(nwReply);
|
|
const CDatabaseReader::JsonDatastoreResponse dsr = this->transformReplyIntoDatastoreResponse(nwReply);
|
|
if (dsr.isSharedFile())
|
|
{
|
|
this->receivedSharedFileHeaderNonClosing(nwReply);
|
|
}
|
|
return dsr;
|
|
}
|
|
|
|
CDbInfoList CDatabaseReader::getDbInfoObjects() const
|
|
{
|
|
static const CDbInfoList e;
|
|
if (!sApp->hasWebDataServices()) { return e; }
|
|
if (!sApp->getWebDataServices()->getDbInfoDataReader()) { return e; }
|
|
return sApp->getWebDataServices()->getDbInfoDataReader()->getInfoObjects();
|
|
}
|
|
|
|
CDbInfoList CDatabaseReader::getSharedInfoObjects() const
|
|
{
|
|
static const CDbInfoList e;
|
|
if (!sApp->hasWebDataServices()) { return e; }
|
|
if (!sApp->getWebDataServices()->getSharedInfoDataReader()) { return e; }
|
|
return sApp->getWebDataServices()->getSharedInfoDataReader()->getInfoObjects();
|
|
}
|
|
|
|
bool CDatabaseReader::hasDbInfoObjects() const
|
|
{
|
|
return getDbInfoObjects().size() > 0;
|
|
}
|
|
|
|
bool CDatabaseReader::hasSharedInfoObjects() const
|
|
{
|
|
return getSharedInfoObjects().size() > 0;
|
|
}
|
|
|
|
bool CDatabaseReader::hasSharedFileHeader(const CEntityFlags::Entity entity) const
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
|
|
return m_sharedFileResponses.contains(entity);
|
|
}
|
|
|
|
bool CDatabaseReader::hasSharedFileHeaders(const CEntityFlags::Entity entities) const
|
|
{
|
|
CEntityFlags::Entity myEntities = maskBySupportedEntities(entities);
|
|
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(myEntities);
|
|
while (currentEntity != CEntityFlags::NoEntity)
|
|
{
|
|
if (!hasSharedFileHeader(currentEntity)) { return false; }
|
|
currentEntity = CEntityFlags::iterateDbEntities(myEntities);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
QDateTime CDatabaseReader::getLatestEntityTimestampFromDbInfoObjects(CEntityFlags::Entity entity) const
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
|
|
static const QDateTime e;
|
|
const CDbInfoList il(getDbInfoObjects());
|
|
if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
|
|
|
|
// for some entities there can be more than one entry because of the
|
|
// raw tables (see DB view last updates)
|
|
const CDbInfo info = il.findFirstByEntityOrDefault(entity);
|
|
if (!info.isValid()) { return e; }
|
|
return info.getUtcTimestamp();
|
|
}
|
|
|
|
QDateTime CDatabaseReader::getLatestEntityTimestampFromSharedInfoObjects(CEntityFlags::Entity entity) const
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
|
|
static const QDateTime e;
|
|
const CDbInfoList il(getSharedInfoObjects());
|
|
if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
|
|
|
|
const CDbInfo info = il.findFirstByEntityOrDefault(entity);
|
|
if (!info.isValid()) { return e; }
|
|
return info.getUtcTimestamp();
|
|
}
|
|
|
|
QDateTime CDatabaseReader::getLatestSharedFileHeaderTimestamp(CEntityFlags::Entity entity) const
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
|
|
static const QDateTime e;
|
|
if (!this->hasSharedFileHeader(entity)) { return e; }
|
|
return m_sharedFileResponses[entity].getLastModifiedTimestamp();
|
|
}
|
|
|
|
bool CDatabaseReader::requestHeadersOfSharedFiles(CEntityFlags::Entity entities)
|
|
{
|
|
if (!this->isInternetAccessible(QString("No network/internet access, will not read shared file headers for %1").arg(CEntityFlags::flagToString(entities)))) { return false; }
|
|
|
|
CEntityFlags::Entity allEntities(this->maskBySupportedEntities(entities));
|
|
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities);
|
|
const CUrl urlSharedData = CGlobalSetup::buildDbDataDirectory(getWorkingDbDataFileLocationUrl());
|
|
|
|
int c = 0;
|
|
while (currentEntity != CEntityFlags::NoEntity)
|
|
{
|
|
const QString fileName = CDbInfo::entityToSharedName(currentEntity);
|
|
Q_ASSERT_X(!fileName.isEmpty(), Q_FUNC_INFO, "No file name for entity");
|
|
CUrl url = urlSharedData;
|
|
url.appendPath(fileName);
|
|
|
|
const QString entityString = CEntityFlags::flagToString(currentEntity);
|
|
CLogMessage(this).info("Triggered read of header for shared file of '%1'") << entityString;
|
|
const QNetworkReply *reply = sApp->headerFromNetwork(url, { this, &CDatabaseReader::receivedSharedFileHeader });
|
|
if (reply) { c++; }
|
|
currentEntity = CEntityFlags::iterateDbEntities(allEntities);
|
|
}
|
|
return c > 0;
|
|
}
|
|
|
|
bool CDatabaseReader::isSharedHeaderNewerThanCacheTimestamp(CEntityFlags::Entity entity) const
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
|
|
const QDateTime cacheTs(this->getCacheTimestamp(entity));
|
|
if (!cacheTs.isValid()) { return true; } // we have no cache ts
|
|
|
|
const QDateTime headerTimestamp(this->getLatestSharedFileHeaderTimestamp(entity));
|
|
if (!headerTimestamp.isValid()) { return false; }
|
|
return headerTimestamp > cacheTs;
|
|
}
|
|
|
|
bool CDatabaseReader::isSharedInfoObjectNewerThanCacheTimestamp(CEntityFlags::Entity entity) const
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
|
|
const QDateTime cacheTs(this->getCacheTimestamp(entity));
|
|
if (!cacheTs.isValid()) { return true; } // we have no cache ts
|
|
|
|
const QDateTime sharedInfoTimestamp(this->getLatestEntityTimestampFromSharedInfoObjects(entity));
|
|
if (!sharedInfoTimestamp.isValid()) { return false; }
|
|
return sharedInfoTimestamp > cacheTs;
|
|
}
|
|
|
|
CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerHeaderTimestamp(CEntityFlags::Entity entities) const
|
|
{
|
|
entities = this->maskBySupportedEntities(entities); // handled by this reader
|
|
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(entities);
|
|
CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity;
|
|
while (currentEntity != CEntityFlags::NoEntity)
|
|
{
|
|
if (this->isSharedHeaderNewerThanCacheTimestamp(currentEntity))
|
|
{
|
|
newerEntities |= currentEntity;
|
|
}
|
|
currentEntity = CEntityFlags::iterateDbEntities(entities);
|
|
}
|
|
return newerEntities;
|
|
}
|
|
|
|
CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerSharedInfoObject(CEntityFlags::Entity entities) const
|
|
{
|
|
entities = this->maskBySupportedEntities(entities); // handled by this reader
|
|
CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(entities);
|
|
CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity;
|
|
while (currentEntity != CEntityFlags::NoEntity)
|
|
{
|
|
if (this->isSharedInfoObjectNewerThanCacheTimestamp(currentEntity))
|
|
{
|
|
newerEntities |= currentEntity;
|
|
}
|
|
currentEntity = CEntityFlags::iterateDbEntities(entities);
|
|
}
|
|
return newerEntities;
|
|
}
|
|
|
|
CDatabaseReaderConfig CDatabaseReader::getConfigForEntity(CEntityFlags::Entity entity) const
|
|
{
|
|
return this->m_config.findFirstOrDefaultForEntity(entity);
|
|
}
|
|
|
|
CEntityFlags::Entity CDatabaseReader::emitReadSignalPerSingleCachedEntity(CEntityFlags::Entity cachedEntities, bool onlyIfHasData)
|
|
{
|
|
if (cachedEntities == CEntityFlags::NoEntity) { return CEntityFlags::NoEntity; }
|
|
CEntityFlags::Entity emitted = CEntityFlags::NoEntity;
|
|
CEntityFlags::Entity cachedEntitiesToEmit = cachedEntities;
|
|
CEntityFlags::Entity currentCachedEntity = CEntityFlags::iterateDbEntities(cachedEntitiesToEmit);
|
|
while (currentCachedEntity)
|
|
{
|
|
const int c = this->getCacheCount(currentCachedEntity);
|
|
if (!onlyIfHasData || c > 0)
|
|
{
|
|
emit dataRead(currentCachedEntity, CEntityFlags::ReadFinished, c);
|
|
emitted |= currentCachedEntity;
|
|
}
|
|
currentCachedEntity = CEntityFlags::iterateDbEntities(cachedEntitiesToEmit);
|
|
}
|
|
return emitted;
|
|
}
|
|
|
|
void CDatabaseReader::emitAndLogDataRead(CEntityFlags::Entity entity, int number, const JsonDatastoreResponse &res)
|
|
{
|
|
// never emit when lock is held, deadlock
|
|
emit dataRead(entity, res.isRestricted() ? CEntityFlags::ReadFinishedRestricted : CEntityFlags::ReadFinished, number);
|
|
CLogMessage(this).info("Read %1 entities of '%2' from '%3' (%4)") << number << CEntityFlags::flagToString(entity) << res.getUrlString() << res.getLoadTimeString();
|
|
}
|
|
|
|
CUrl CDatabaseReader::getBaseUrl(CDbFlags::DataRetrievalModeFlag mode) const
|
|
{
|
|
Q_ASSERT_X(sApp, Q_FUNC_INFO, "Missing app object, URLs cannot be obtained");
|
|
switch (mode)
|
|
{
|
|
case CDbFlags::DbReading:
|
|
return this->getDbServiceBaseUrl().withAppendedPath("/service");
|
|
case CDbFlags::SharedInfoOnly:
|
|
case CDbFlags::Shared:
|
|
return CDatabaseReader::getWorkingDbDataFileLocationUrl();
|
|
default:
|
|
qFatal("Wrong mode");
|
|
break;
|
|
}
|
|
return CUrl();
|
|
}
|
|
|
|
bool CDatabaseReader::isChangedUrl(const CUrl &oldUrl, const CUrl ¤tUrl)
|
|
{
|
|
if (oldUrl.isEmpty()) { return true; }
|
|
Q_ASSERT_X(!currentUrl.isEmpty(), Q_FUNC_INFO, "No base URL");
|
|
|
|
const QString old(oldUrl.getFullUrl(false));
|
|
const QString current(currentUrl.getFullUrl(false));
|
|
return old != current;
|
|
}
|
|
|
|
void CDatabaseReader::receivedSharedFileHeader(QNetworkReply *nwReplyPtr)
|
|
{
|
|
// wrap pointer, make sure any exit cleans up reply
|
|
// required to use delete later as object is created in a different thread
|
|
QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
|
|
if (!this->doWorkCheck()) { return; }
|
|
this->receivedSharedFileHeaderNonClosing(nwReplyPtr);
|
|
nwReply->close();
|
|
}
|
|
|
|
void CDatabaseReader::receivedSharedFileHeaderNonClosing(QNetworkReply *nwReply)
|
|
{
|
|
if (this->isAbandoned()) { return; }
|
|
|
|
const HeaderResponse headerResponse = this->transformReplyIntoHeaderResponse(nwReply);
|
|
const QString fileName = nwReply->url().fileName();
|
|
const CEntityFlags::Entity entity = CEntityFlags::singleEntityByName(fileName);
|
|
this->m_sharedFileResponses[entity] = headerResponse;
|
|
|
|
CLogMessage(this).info("Received header for shared file of '%1' from '%2'") << fileName << headerResponse.getUrl().toQString();
|
|
emit this->sharedFileHeaderRead(entity, fileName, !headerResponse.hasWarningOrAboveMessage());
|
|
}
|
|
|
|
bool CDatabaseReader::hasReceivedOkReply() const
|
|
{
|
|
QReadLocker rl(&this->m_statusLock);
|
|
return m_1stReplyReceived && m_1stReplyStatus == QNetworkReply::NoError;
|
|
}
|
|
|
|
bool CDatabaseReader::hasReceivedOkReply(QString &message) const
|
|
{
|
|
QReadLocker rl(&this->m_statusLock);
|
|
message = m_statusMessage;
|
|
return m_1stReplyReceived && m_1stReplyStatus == QNetworkReply::NoError;
|
|
}
|
|
|
|
bool CDatabaseReader::hasReceivedFirstReply() const
|
|
{
|
|
QReadLocker rl(&this->m_statusLock);
|
|
return m_1stReplyReceived;
|
|
}
|
|
|
|
CEntityFlags::Entity CDatabaseReader::maskBySupportedEntities(CEntityFlags::Entity entities) const
|
|
{
|
|
return entities & getSupportedEntities();
|
|
}
|
|
|
|
bool CDatabaseReader::supportsAnyOfEntities(CEntityFlags::Entity entities) const
|
|
{
|
|
return static_cast<int>(maskBySupportedEntities(entities)) > 0;
|
|
}
|
|
|
|
const QString &CDatabaseReader::getStatusMessage() const
|
|
{
|
|
return this->m_statusMessage;
|
|
}
|
|
|
|
void CDatabaseReader::setReplyStatus(QNetworkReply::NetworkError status, const QString &message)
|
|
{
|
|
QWriteLocker wl(&this->m_statusLock);
|
|
this->m_statusMessage = message;
|
|
this->m_1stReplyStatus = status;
|
|
this->m_1stReplyReceived = true;
|
|
}
|
|
|
|
void CDatabaseReader::setReplyStatus(QNetworkReply *nwReply)
|
|
{
|
|
Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing network reply");
|
|
if (nwReply && nwReply->isFinished())
|
|
{
|
|
this->setReplyStatus(nwReply->error(), nwReply->errorString());
|
|
}
|
|
}
|
|
|
|
QString CDatabaseReader::fileNameForMode(CEntityFlags::Entity entity, CDbFlags::DataRetrievalModeFlag mode)
|
|
{
|
|
Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "needs single entity");
|
|
switch (mode)
|
|
{
|
|
case CDbFlags::Shared:
|
|
return CDbInfo::entityToSharedName(entity);
|
|
case CDbFlags::SharedInfoOnly:
|
|
return CDbInfo::sharedInfoFileName();
|
|
default:
|
|
case CDbFlags::DbReading:
|
|
return CDbInfo::entityToServiceName(entity);
|
|
}
|
|
}
|
|
|
|
QString CDatabaseReader::dateTimeToDbLatestTs(const QDateTime &ts)
|
|
{
|
|
if (!ts.isValid()) return "";
|
|
return ts.toUTC().toString(Qt::ISODate);
|
|
}
|
|
|
|
const CLogCategoryList &CDatabaseReader::getLogCategories()
|
|
{
|
|
static const BlackMisc::CLogCategoryList cats
|
|
(
|
|
CThreadedReader::getLogCategories().join({ BlackMisc::CLogCategory::swiftDbWebservice(), BlackMisc::CLogCategory::webservice() })
|
|
);
|
|
return cats;
|
|
}
|
|
|
|
const QString &CDatabaseReader::parameterLatestTimestamp()
|
|
{
|
|
static const QString p("latestTimestamp");
|
|
return p;
|
|
}
|
|
|
|
QString CDatabaseReader::queryLatestTimestamp(const QDateTime &ts)
|
|
{
|
|
if (!ts.isValid()) return "";
|
|
const QString q = parameterLatestTimestamp() + "=" + dateTimeToDbLatestTs(ts);
|
|
return q;
|
|
}
|
|
|
|
const QString &CDatabaseReader::parameterLatestId()
|
|
{
|
|
static const QString p("latestId");
|
|
return p;
|
|
}
|
|
|
|
const CUrl &CDatabaseReader::getDbUrl()
|
|
{
|
|
static const CUrl dbUrl(sApp->getGlobalSetup().getDbRootDirectoryUrl());
|
|
return dbUrl;
|
|
}
|
|
|
|
CUrl CDatabaseReader::getWorkingDbDataFileLocationUrl()
|
|
{
|
|
return CDatabaseReader::s_workingSharedDbData;
|
|
}
|
|
|
|
void CDatabaseReader::cacheHasChanged(CEntityFlags::Entity entities)
|
|
{
|
|
this->emitReadSignalPerSingleCachedEntity(entities, false);
|
|
}
|
|
|
|
bool CDatabaseReader::canPingSwiftServer()
|
|
{
|
|
const CUrl url(getDbUrl());
|
|
return CNetworkUtils::canConnect(url);
|
|
}
|
|
|
|
bool CDatabaseReader::initWorkingUrls(bool force)
|
|
{
|
|
if (!force && !CDatabaseReader::s_workingSharedDbData.isEmpty()) { return false; }
|
|
CDatabaseReader::s_workingSharedDbData = sApp->getGlobalSetup().getSwiftDbDataFileLocationUrls().getRandomWorkingUrl();
|
|
return !CDatabaseReader::s_workingSharedDbData.isEmpty();
|
|
}
|
|
|
|
CUrl CDatabaseReader::getCurrentSharedDbDataUrl()
|
|
{
|
|
return CDatabaseReader::s_workingSharedDbData;
|
|
}
|
|
|
|
void CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent, JsonDatastoreResponse &datastoreResponse)
|
|
{
|
|
if (jsonContent.isEmpty())
|
|
{
|
|
datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty string, no data"));
|
|
return;
|
|
}
|
|
|
|
const QJsonDocument jsonResponse = CDatabaseUtils::databaseJsonToQJsonDocument(jsonContent);
|
|
if (jsonResponse.isEmpty())
|
|
{
|
|
datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty JSON, no data"));
|
|
return;
|
|
}
|
|
|
|
if (jsonResponse.isArray())
|
|
{
|
|
// directly an array, no further info
|
|
datastoreResponse.setJsonArray(jsonResponse.array());
|
|
datastoreResponse.setLastModifiedTimestamp(QDateTime::currentDateTimeUtc());
|
|
}
|
|
else
|
|
{
|
|
const QJsonObject responseObject(jsonResponse.object());
|
|
datastoreResponse.setJsonArray(responseObject["data"].toArray());
|
|
const QString ts(responseObject["latest"].toString());
|
|
datastoreResponse.setLastModifiedTimestamp(ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts));
|
|
datastoreResponse.setRestricted(responseObject["restricted"].toBool());
|
|
}
|
|
}
|
|
|
|
void CDatabaseReader::JsonDatastoreResponse::setJsonArray(const QJsonArray &value)
|
|
{
|
|
m_jsonArray = value;
|
|
m_arraySize = value.size();
|
|
}
|
|
|
|
bool CDatabaseReader::HeaderResponse::isSharedFile() const
|
|
{
|
|
const QString fn(getUrl().getFileName());
|
|
return CDbInfo::sharedFileNames().contains(fn, Qt::CaseInsensitive);
|
|
}
|
|
} // ns
|
|
} // ns
|