From 77c539b6501a198760e12a51e9ea79e80519c41a Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Thu, 20 Apr 2017 02:35:29 +0200 Subject: [PATCH] Ref T24, adjusted readers * utility functions for shared info objects (count/timestamp) * support for shared and DB info objects * renamed functions reflecting using info objects now (no longer headers) --- src/blackcore/db/databasereader.cpp | 89 ++++++++++++++++------- src/blackcore/db/databasereader.h | 39 +++++++--- src/blackcore/db/databasereaderconfig.cpp | 14 ++-- src/blackcore/db/databasereaderconfig.h | 8 +- src/blackcore/db/infodatareader.cpp | 53 ++++++++++---- src/blackcore/db/infodatareader.h | 28 ++++--- 6 files changed, 159 insertions(+), 72 deletions(-) diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp index e4a95ce03..9e6409d51 100644 --- a/src/blackcore/db/databasereader.cpp +++ b/src/blackcore/db/databasereader.cpp @@ -52,8 +52,8 @@ namespace BlackCore if (this->isShuttingDown()) { return; } // we accept cached cached data - Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly"); - const bool hasInfoObjects = this->hasInfoObjects(); // no info objects is not necessarily error, but indicates a) either data not available (DB down) or b) only caches are used + Q_ASSERT_X(!entities.testFlag(CEntityFlags::DbInfoObjectEntity), Q_FUNC_INFO, "Read info objects directly"); + const bool hasInfoObjects = this->hasDbInfoObjects(); // no info objects is not necessarily an error, but indicates a) either data not available (DB down) or b) only caches are used CEntityFlags::Entity allEntities = entities; CEntityFlags::Entity cachedEntities = CEntityFlags::NoEntity; CEntityFlags::Entity dbEntities = CEntityFlags::NoEntity; @@ -74,7 +74,7 @@ namespace BlackCore { const bool changedUrl = this->hasChangedUrl(currentEntity); const QDateTime cacheTs(this->getCacheTimestamp(currentEntity)); - const QDateTime latestEntityTs(this->getLatestEntityTimestampFromInfoObjects(currentEntity)); + const QDateTime latestEntityTs(this->getLatestEntityTimestampFromDbInfoObjects(currentEntity)); const qint64 cacheTimestamp = cacheTs.isValid() ? cacheTs.toMSecsSinceEpoch() : -1; const qint64 latestEntityTimestamp = latestEntityTs.isValid() ? latestEntityTs.toMSecsSinceEpoch() : -1; Q_ASSERT_X(latestEntityTimestamp >= 0, Q_FUNC_INFO, "Missing timestamp"); @@ -265,17 +265,30 @@ namespace BlackCore return dsr; } - CDbInfoList CDatabaseReader::infoList() const + CDbInfoList CDatabaseReader::getDbInfoObjects() const { static const CDbInfoList e; if (!sApp->hasWebDataServices()) { return e; } if (!sApp->getWebDataServices()->getDbInfoDataReader()) { return e; } - return sApp->getWebDataServices()->getDbInfoDataReader()->getDbInfoObjects(); + return sApp->getWebDataServices()->getDbInfoDataReader()->getInfoObjects(); } - bool CDatabaseReader::hasInfoObjects() const + CDbInfoList CDatabaseReader::getSharedInfoObjects() const { - return infoList().size() > 0; + static const CDbInfoList e; + if (!sApp->hasWebDataServices()) { return e; } + if (!sApp->getWebDataServices()->getSharedInfoDataReader()) { return e; } + return sApp->getWebDataServices()->getSharedInfoDataReader()->getInfoObjects(); + } + + bool CDatabaseReader::hasDbInfoObjects() const + { + return getDbInfoObjects().size() > 0; + } + + bool CDatabaseReader::hasSharedInfoObjects() const + { + return getSharedInfoObjects().size() > 0; } bool CDatabaseReader::hasSharedFileHeader(const CEntityFlags::Entity entity) const @@ -296,11 +309,11 @@ namespace BlackCore return true; } - QDateTime CDatabaseReader::getLatestEntityTimestampFromInfoObjects(CEntityFlags::Entity entity) const + QDateTime CDatabaseReader::getLatestEntityTimestampFromDbInfoObjects(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); static const QDateTime e; - const CDbInfoList il(infoList()); + const CDbInfoList il(getDbInfoObjects()); if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; } // for some entities there can be more than one entry because of the @@ -310,6 +323,18 @@ namespace BlackCore return info.getUtcTimestamp(); } + QDateTime CDatabaseReader::getLatestEntityTimestampFromSharedInfoObjects(CEntityFlags::Entity entity) const + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); + static const QDateTime e; + const CDbInfoList il(getSharedInfoObjects()); + if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; } + + const CDbInfo info = il.findFirstByEntityOrDefault(entity); + if (!info.isValid()) { return e; } + return info.getUtcTimestamp(); + } + QDateTime CDatabaseReader::getLatestSharedFileHeaderTimestamp(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); @@ -355,17 +380,20 @@ namespace BlackCore const QDateTime headerTimestamp(this->getLatestSharedFileHeaderTimestamp(entity)); if (!headerTimestamp.isValid()) { return false; } - - // we restrict by latest entity from info objects (if available) - // a header ts shall be never newer than the info object, if it is it is an issue with shared file sync - const QDateTime infoObjectTs = this->getLatestEntityTimestampFromInfoObjects(entity); - if (infoObjectTs.isValid() && infoObjectTs < headerTimestamp) - { - return infoObjectTs > cacheTs; - } return headerTimestamp > cacheTs; } + bool CDatabaseReader::isSharedInfoObjectNewerThanCacheTimestamp(CEntityFlags::Entity entity) const + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity"); + const QDateTime cacheTs(this->getCacheTimestamp(entity)); + if (!cacheTs.isValid()) { return true; } // we have no cache ts + + const QDateTime sharedInfoTimestamp(this->getLatestSharedFileHeaderTimestamp(entity)); + if (!sharedInfoTimestamp.isValid()) { return false; } + return sharedInfoTimestamp > cacheTs; + } + CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerHeaderTimestamp(CEntityFlags::Entity entities) const { entities = this->maskBySupportedEntities(entities); // handled by this reader @@ -382,14 +410,20 @@ namespace BlackCore return newerEntities; } - int CDatabaseReader::getCountFromInfoObjects(CEntityFlags::Entity entity) const + CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerSharedInfoObject(CEntityFlags::Entity entities) const { - static const QDateTime e; - const CDbInfoList il(infoList()); - if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return -1; } - CDbInfo info = il.findFirstByEntityOrDefault(entity); - if (!info.isValid()) { return -1; } - return info.getEntries(); + entities = this->maskBySupportedEntities(entities); // handled by this reader + CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(entities); + CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity; + while (currentEntity != CEntityFlags::NoEntity) + { + if (this->isSharedInfoObjectNewerThanCacheTimestamp(currentEntity)) + { + newerEntities |= currentEntity; + } + currentEntity = CEntityFlags::iterateDbEntities(entities); + } + return newerEntities; } CDatabaseReaderConfig CDatabaseReader::getConfigForEntity(CEntityFlags::Entity entity) const @@ -411,7 +445,7 @@ namespace BlackCore emit dataRead(currentCachedEntity, CEntityFlags::ReadFinished, c); emitted |= currentCachedEntity; } - currentCachedEntity = CEntityFlags::iterateDbEntities(cachedEntitiesToEmit); + currentCachedEntity = CEntityFlags::iterateDbEntities(cachedEntitiesToEmit); } return emitted; } @@ -431,7 +465,7 @@ namespace BlackCore { case CDbFlags::DbReading: return this->getDbServiceBaseUrl().withAppendedPath("/service"); - case CDbFlags::SharedHeadersOnly: + case CDbFlags::SharedInfoOnly: case CDbFlags::Shared: return CDatabaseReader::getWorkingDbDataFileLocationUrl(); default: @@ -531,8 +565,9 @@ namespace BlackCore switch (mode) { case CDbFlags::Shared: - case CDbFlags::SharedHeadersOnly: return CDbInfo::entityToSharedName(entity); + case CDbFlags::SharedInfoOnly: + return CDbInfo::sharedInfoFileName(); default: case CDbFlags::DbReading: return CDbInfo::entityToServiceName(entity); diff --git a/src/blackcore/db/databasereader.h b/src/blackcore/db/databasereader.h index cd5a2ebaa..2abe6776f 100644 --- a/src/blackcore/db/databasereader.h +++ b/src/blackcore/db/databasereader.h @@ -181,8 +181,11 @@ namespace BlackCore //! Cache`s number of entities virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const = 0; - //! Info objects available? - bool hasInfoObjects() const; + //! DB info objects available? + bool hasDbInfoObjects() const; + + //! Shared info objects available? + bool hasSharedInfoObjects() const; //! Header of shared file read (for single entity)? bool hasSharedFileHeader(const BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -190,26 +193,34 @@ namespace BlackCore //! Headers of shared file read (for single entity)? bool hasSharedFileHeaders(const BlackMisc::Network::CEntityFlags::Entity entities) const; - //! Obtain latest object timestamp from info objects + //! Obtain latest object timestamp from DB info objects //! \sa BlackCore::Db::CInfoDataReader - QDateTime getLatestEntityTimestampFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; + QDateTime getLatestEntityTimestampFromDbInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Obtain latest object timestamp from shared info objects + //! \sa BlackCore::Db::CInfoDataReader + QDateTime getLatestEntityTimestampFromSharedInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; //! Header timestamp (last-modified) for shared file + //! \deprecated use getLatestEntityTimestampFromSharedInfoObjects QDateTime getLatestSharedFileHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; - //! Is the file timestamp neer than cache timestamp? + //! Is the file timestamp newer than cache timestamp? + //! \deprecated use isSharedInfoNewerThanCacheTimestamp bool isSharedHeaderNewerThanCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! Is the shared info timestamp newer than cache timestamp? + bool isSharedInfoObjectNewerThanCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! Those entities where the timestamp of header is newer than the cache timestamp BlackMisc::Network::CEntityFlags::Entity getEntitesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const; + //! Those entities where the timestamp of shared info obejct is newer than the cache timestamp + BlackMisc::Network::CEntityFlags::Entity getEntitesWithNewerSharedInfoObject(BlackMisc::Network::CEntityFlags::Entity entities) const; + //! Request header of shared file bool requestHeadersOfSharedFiles(BlackMisc::Network::CEntityFlags::Entity entities); - //! Count from info objects - //! \sa BlackCore::Db::CInfoDataReader - int getCountFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; - //! Status message (error message) const QString &getStatusMessage() const; @@ -256,9 +267,13 @@ namespace BlackCore //! Check if terminated or error, otherwise split into array of objects CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply); - //! Info list (latest data timestamp) + //! DB Info list (latest data timestamps from DB web service) //! \sa BlackCore::Db::CInfoDataReader - BlackMisc::Db::CDbInfoList infoList() const; + BlackMisc::Db::CDbInfoList getDbInfoObjects() const; + + //! Shared info list (latest data timestamps from DB web service) + //! \sa BlackCore::Db::CInfoDataReader + BlackMisc::Db::CDbInfoList getSharedInfoObjects() const; //! Config for given entity CDatabaseReaderConfig getConfigForEntity(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -307,7 +322,7 @@ namespace BlackCore //! @} private: - static BlackMisc::Network::CUrl s_workingSharedDbData; //!< one chhosen URL for all DB reader objects + static BlackMisc::Network::CUrl s_workingSharedDbData; //!< one choosen URL for all DB reader objects //! Start reading in own thread (without config/caching) //! \remarks can handle DB or shared file reads diff --git a/src/blackcore/db/databasereaderconfig.cpp b/src/blackcore/db/databasereaderconfig.cpp index 2d9df6f86..918445f5a 100644 --- a/src/blackcore/db/databasereaderconfig.cpp +++ b/src/blackcore/db/databasereaderconfig.cpp @@ -74,14 +74,14 @@ namespace BlackCore return (this->getRetrievalMode().testFlag(CDbFlags::DbReading)); } - bool CDatabaseReaderConfig::needsSharedHeader() const + bool CDatabaseReaderConfig::needsSharedInfoFile() const { if (!this->isValid()) { return false; } if (!CEntityFlags::anySwiftDbEntity(this->getEntities())) { return false; } - return (this->getRetrievalMode().testFlag(CDbFlags::Shared) || this->getRetrievalMode().testFlag(CDbFlags::SharedHeadersOnly)); + return (this->getRetrievalMode().testFlag(CDbFlags::Shared) || this->getRetrievalMode().testFlag(CDbFlags::SharedInfoOnly)); } - bool CDatabaseReaderConfig::needsSharedHeaderLoaded() const + bool CDatabaseReaderConfig::needsSharedInfoFileLoaded() const { if (!this->isValid()) { return false; } if (!CEntityFlags::anySwiftDbEntity(this->getEntities())) { return false; } @@ -162,22 +162,22 @@ namespace BlackCore return false; } - bool CDatabaseReaderConfigList::needsSharedHeaders(CEntityFlags::Entity entities) const + bool CDatabaseReaderConfigList::needsSharedInfoObjects(CEntityFlags::Entity entities) const { for (const CDatabaseReaderConfig &config : *this) { if (!config.supportsEntities(entities)) { continue; } - if (config.needsSharedHeader()) { return true; } + if (config.needsSharedInfoFile()) { return true; } } return false; } - bool CDatabaseReaderConfigList::needsSharedHeadersLoaded(CEntityFlags::Entity entities) const + bool CDatabaseReaderConfigList::needsSharedInfoFileLoaded(CEntityFlags::Entity entities) const { for (const CDatabaseReaderConfig &config : *this) { if (!config.supportsEntities(entities)) { continue; } - if (config.needsSharedHeaderLoaded()) { return true; } + if (config.needsSharedInfoFileLoaded()) { return true; } } return false; } diff --git a/src/blackcore/db/databasereaderconfig.h b/src/blackcore/db/databasereaderconfig.h index 1a994ec47..037e4cb9a 100644 --- a/src/blackcore/db/databasereaderconfig.h +++ b/src/blackcore/db/databasereaderconfig.h @@ -66,10 +66,10 @@ namespace BlackCore bool possiblyReadsFromSwiftDb() const; //! Needs the shared header - bool needsSharedHeader() const; + bool needsSharedInfoFile() const; //! Needs the shared header loaded before it can be continued - bool needsSharedHeaderLoaded() const; + bool needsSharedInfoFileLoaded() const; //! Will write to swift DB bool possiblyWritesToSwiftDb() const; @@ -122,10 +122,10 @@ namespace BlackCore bool possiblyWritesToSwiftDb() const; //! Needs any shared header - bool needsSharedHeaders(BlackMisc::Network::CEntityFlags::Entity entities) const; + bool needsSharedInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities) const; //! Needs any shared header loaded before continued - bool needsSharedHeadersLoaded(BlackMisc::Network::CEntityFlags::Entity entities) const; + bool needsSharedInfoFileLoaded(BlackMisc::Network::CEntityFlags::Entity entities) const; //! Entities which will use cache or DB, so no canceled or ignored ones BlackMisc::Network::CEntityFlags::Entity getEntitesCachedOrReadFromDB() const; diff --git a/src/blackcore/db/infodatareader.cpp b/src/blackcore/db/infodatareader.cpp index 87b50c3ec..8132fe7b8 100644 --- a/src/blackcore/db/infodatareader.cpp +++ b/src/blackcore/db/infodatareader.cpp @@ -26,17 +26,19 @@ namespace BlackCore { namespace Db { - CInfoDataReader::CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config) : - CDatabaseReader(owner, config, "CInfoDataReader") - { } + CInfoDataReader::CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config, CDbFlags::DataRetrievalModeFlag mode) : + CDatabaseReader(owner, config, "CInfoDataReader"), m_mode(mode) + { + Q_ASSERT_X(mode == CDbFlags::DbReading || mode == CDbFlags::Shared, Q_FUNC_INFO, "Wrong mode"); + } - CDbInfoList CInfoDataReader::getDbInfoObjects() const + CDbInfoList CInfoDataReader::getInfoObjects() const { QReadLocker l(&m_lockInfoObjects); return m_infoObjects; } - int CInfoDataReader::getDbInfoObjectCount() const + int CInfoDataReader::getInfoObjectCount() const { QReadLocker l(&m_lockInfoObjects); return m_infoObjects.size(); @@ -44,7 +46,7 @@ namespace BlackCore bool CInfoDataReader::areAllDataRead() const { - return getDbInfoObjectCount() > 4; + return getInfoObjectCount() > 4; } void CInfoDataReader::synchronizeCaches(CEntityFlags::Entity entities) @@ -93,15 +95,15 @@ namespace BlackCore void CInfoDataReader::read() { - const CUrl url(getDbInfoObjectsUrl()); + const CUrl url(this->getInfoObjectsUrl()); if (!url.isEmpty()) { sApp->getFromNetwork(url, { this, &CInfoDataReader::ps_parseInfoObjectsData}); - emit dataRead(CEntityFlags::InfoObjectEntity, CEntityFlags::StartRead, 0); + emit dataRead(this->getEntityForMode(), CEntityFlags::StartRead, 0); } else { - CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::InfoObjectEntity); + CLogMessage(this).error("No URL for '%1'") << CEntityFlags::flagToString(this->getEntityForMode()); } } @@ -112,11 +114,11 @@ namespace BlackCore QScopedPointer nwReply(nwReplyPtr); if (this->isShuttingDown()) { return; } - CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); + const CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data()); if (res.hasErrorMessage()) { CLogMessage::preformatted(res.lastWarningOrAbove()); - emit dataRead(CEntityFlags::InfoObjectEntity, CEntityFlags::ReadFailed, 0); + emit dataRead(this->getEntityForMode(), CEntityFlags::ReadFailed, 0); return; } @@ -130,7 +132,7 @@ namespace BlackCore this->m_infoObjects = infoObjects; } - this->emitAndLogDataRead(CEntityFlags::InfoObjectEntity, n, res); + this->emitAndLogDataRead(this->getEntityForMode(), n, res); } CUrl CInfoDataReader::getDbInfoObjectsUrl() const @@ -138,9 +140,34 @@ namespace BlackCore return getBaseUrl(CDbFlags::DbReading).withAppendedPath("jsondbinfo.php"); } + CUrl CInfoDataReader::getSharedInfoObjectsUrl() const + { + return getBaseUrl(CDbFlags::Shared).withAppendedPath(CDbInfo::sharedInfoFileName()); + } + + CEntityFlags::EntityFlag CInfoDataReader::getEntityForMode() const + { + if (this->m_mode == CDbFlags::DbReading) return CEntityFlags::DbInfoObjectEntity; + if (this->m_mode == CDbFlags::Shared) return CEntityFlags::SharedInfoObjectEntity; + qFatal("Wrong mode"); + return CEntityFlags::NoEntity; + } + + CUrl CInfoDataReader::getInfoObjectsUrl() const + { + switch (m_mode) + { + case CDbFlags::DbReading: return getDbInfoObjectsUrl(); + case CDbFlags::Shared: return getSharedInfoObjectsUrl(); + default: + qFatal("Wrong mode"); + } + return CUrl(); + } + CEntityFlags::Entity CInfoDataReader::getSupportedEntities() const { - return CEntityFlags::InfoObjectEntity; + return this->getEntityForMode(); } } // namespace } // namespace diff --git a/src/blackcore/db/infodatareader.h b/src/blackcore/db/infodatareader.h index dbe1186e0..6260184cd 100644 --- a/src/blackcore/db/infodatareader.h +++ b/src/blackcore/db/infodatareader.h @@ -24,33 +24,33 @@ namespace BlackCore { namespace Db { - //! Read information about data from Database + //! Read information about data from Database or shared file class BLACKCORE_EXPORT CInfoDataReader : public CDatabaseReader { Q_OBJECT public: //! Constructor - explicit CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config); + explicit CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config, BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode); - //! Get info list + //! Get info list (either shared or from DB) //! \threadsafe - BlackMisc::Db::CDbInfoList getDbInfoObjects() const; + BlackMisc::Db::CDbInfoList getInfoObjects() const; - //! Get info list size + //! Get info list size (either shared or from DB) //! \threadsafe - int getDbInfoObjectCount() const; + int getInfoObjectCount() const; //! All data read? //! \threadsafe bool areAllDataRead() const; - //! URL info objects web service - BlackMisc::Network::CUrl getDbInfoObjectsUrl() const; - //! Allow to call directly, special for info objects reader void read(); + //! URL depending on mode + BlackMisc::Network::CUrl getInfoObjectsUrl() const; + // cache handling for base class: no cache handling here in that case virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override; virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const override; @@ -69,6 +69,16 @@ namespace BlackCore void ps_parseInfoObjectsData(QNetworkReply *nwReply); private: + //! URL info objects web service + BlackMisc::Network::CUrl getDbInfoObjectsUrl() const; + + //! URL shared info objects + BlackMisc::Network::CUrl getSharedInfoObjectsUrl() const; + + //! Entity for mode + BlackMisc::Network::CEntityFlags::EntityFlag getEntityForMode() const; + + BlackMisc::Db::CDbFlags::DataRetrievalModeFlag m_mode; //!< shared or DB web service? BlackMisc::Db::CDbInfoList m_infoObjects; BlackMisc::Network::CUrl m_urlInfoObjects; mutable QReadWriteLock m_lockInfoObjects;