Ref T24, adjusted readers

* utility functions for shared info objects (count/timestamp)
* support for shared and DB info objects
* renamed functions reflecting using info objects now (no longer headers)

src/blackcore/db/databasereader.cpp

@@ -52,8 +52,8 @@ namespace BlackCore
             if (this->isShuttingDown()) { return; }
 
             // we accept cached cached data
-            Q_ASSERT_X(!entities.testFlag(CEntityFlags::InfoObjectEntity), Q_FUNC_INFO, "Read info objects directly");
+            Q_ASSERT_X(!entities.testFlag(CEntityFlags::DbInfoObjectEntity), Q_FUNC_INFO, "Read info objects directly");
-            const bool hasInfoObjects = this->hasInfoObjects(); // no info objects is not necessarily error, but indicates a) either data not available (DB down) or b) only caches are used
+            const bool hasInfoObjects = this->hasDbInfoObjects(); // no info objects is not necessarily an error, but indicates a) either data not available (DB down) or b) only caches are used
             CEntityFlags::Entity allEntities    = entities;
             CEntityFlags::Entity cachedEntities = CEntityFlags::NoEntity;
             CEntityFlags::Entity dbEntities     = CEntityFlags::NoEntity;
@@ -74,7 +74,7 @@ namespace BlackCore
                     {
                         const bool changedUrl = this->hasChangedUrl(currentEntity);
                         const QDateTime cacheTs(this->getCacheTimestamp(currentEntity));
-                        const QDateTime latestEntityTs(this->getLatestEntityTimestampFromInfoObjects(currentEntity));
+                        const QDateTime latestEntityTs(this->getLatestEntityTimestampFromDbInfoObjects(currentEntity));
                         const qint64 cacheTimestamp = cacheTs.isValid() ? cacheTs.toMSecsSinceEpoch() : -1;
                         const qint64 latestEntityTimestamp = latestEntityTs.isValid() ? latestEntityTs.toMSecsSinceEpoch() : -1;
                         Q_ASSERT_X(latestEntityTimestamp >= 0, Q_FUNC_INFO, "Missing timestamp");
@@ -265,17 +265,30 @@ namespace BlackCore
             return dsr;
         }
 
-        CDbInfoList CDatabaseReader::infoList() const
+        CDbInfoList CDatabaseReader::getDbInfoObjects() const
         {
             static const CDbInfoList e;
             if (!sApp->hasWebDataServices()) { return e; }
             if (!sApp->getWebDataServices()->getDbInfoDataReader()) { return e; }
-            return sApp->getWebDataServices()->getDbInfoDataReader()->getDbInfoObjects();
+            return sApp->getWebDataServices()->getDbInfoDataReader()->getInfoObjects();
         }
 
-        bool CDatabaseReader::hasInfoObjects() const
+        CDbInfoList CDatabaseReader::getSharedInfoObjects() const
         {
-            return infoList().size() > 0;
+            static const CDbInfoList e;
+            if (!sApp->hasWebDataServices()) { return e; }
+            if (!sApp->getWebDataServices()->getSharedInfoDataReader()) { return e; }
+            return sApp->getWebDataServices()->getSharedInfoDataReader()->getInfoObjects();
+        }
+
+        bool CDatabaseReader::hasDbInfoObjects() const
+        {
+            return getDbInfoObjects().size() > 0;
+        }
+
+        bool CDatabaseReader::hasSharedInfoObjects() const
+        {
+            return getSharedInfoObjects().size() > 0;
         }
 
         bool CDatabaseReader::hasSharedFileHeader(const CEntityFlags::Entity entity) const
@@ -296,11 +309,11 @@ namespace BlackCore
             return true;
         }
 
-        QDateTime CDatabaseReader::getLatestEntityTimestampFromInfoObjects(CEntityFlags::Entity entity) const
+        QDateTime CDatabaseReader::getLatestEntityTimestampFromDbInfoObjects(CEntityFlags::Entity entity) const
         {
             Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
             static const QDateTime e;
-            const CDbInfoList il(infoList());
+            const CDbInfoList il(getDbInfoObjects());
             if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
 
             // for some entities there can be more than one entry because of the
@@ -310,6 +323,18 @@ namespace BlackCore
             return info.getUtcTimestamp();
         }
 
+        QDateTime CDatabaseReader::getLatestEntityTimestampFromSharedInfoObjects(CEntityFlags::Entity entity) const
+        {
+            Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
+            static const QDateTime e;
+            const CDbInfoList il(getSharedInfoObjects());
+            if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
+
+            const CDbInfo info = il.findFirstByEntityOrDefault(entity);
+            if (!info.isValid()) { return e; }
+            return info.getUtcTimestamp();
+        }
+
         QDateTime CDatabaseReader::getLatestSharedFileHeaderTimestamp(CEntityFlags::Entity entity) const
         {
             Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
@@ -355,17 +380,20 @@ namespace BlackCore
 
             const QDateTime headerTimestamp(this->getLatestSharedFileHeaderTimestamp(entity));
             if (!headerTimestamp.isValid()) { return false; }
-
-            // we restrict by latest entity from info objects (if available)
-            // a header ts shall be never newer than the info object, if it is it is an issue with shared file sync
-            const QDateTime infoObjectTs = this->getLatestEntityTimestampFromInfoObjects(entity);
-            if (infoObjectTs.isValid() && infoObjectTs < headerTimestamp)
-            {
-                return infoObjectTs > cacheTs;
-            }
             return headerTimestamp > cacheTs;
         }
 
+        bool CDatabaseReader::isSharedInfoObjectNewerThanCacheTimestamp(CEntityFlags::Entity entity) const
+        {
+            Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "need single entity");
+            const QDateTime cacheTs(this->getCacheTimestamp(entity));
+            if (!cacheTs.isValid()) { return true; } // we have no cache ts
+
+            const QDateTime sharedInfoTimestamp(this->getLatestSharedFileHeaderTimestamp(entity));
+            if (!sharedInfoTimestamp.isValid()) { return false; }
+            return sharedInfoTimestamp > cacheTs;
+        }
+
         CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerHeaderTimestamp(CEntityFlags::Entity entities) const
         {
             entities = this->maskBySupportedEntities(entities); // handled by this reader
@@ -382,14 +410,20 @@ namespace BlackCore
             return newerEntities;
         }
 
-        int CDatabaseReader::getCountFromInfoObjects(CEntityFlags::Entity entity) const
+        CEntityFlags::Entity CDatabaseReader::getEntitesWithNewerSharedInfoObject(CEntityFlags::Entity entities) const
         {
-            static const QDateTime e;
+            entities = this->maskBySupportedEntities(entities); // handled by this reader
-            const CDbInfoList il(infoList());
+            CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(entities);
-            if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return -1; }
+            CEntityFlags::Entity newerEntities = CEntityFlags::NoEntity;
-            CDbInfo info = il.findFirstByEntityOrDefault(entity);
+            while (currentEntity != CEntityFlags::NoEntity)
-            if (!info.isValid()) { return -1; }
+            {
-            return info.getEntries();
+                if (this->isSharedInfoObjectNewerThanCacheTimestamp(currentEntity))
+                {
+                    newerEntities |= currentEntity;
+                }
+                currentEntity = CEntityFlags::iterateDbEntities(entities);
+            }
+            return newerEntities;
         }
 
         CDatabaseReaderConfig CDatabaseReader::getConfigForEntity(CEntityFlags::Entity entity) const
@@ -431,7 +465,7 @@ namespace BlackCore
             {
             case CDbFlags::DbReading:
                 return this->getDbServiceBaseUrl().withAppendedPath("/service");
-            case CDbFlags::SharedHeadersOnly:
+            case CDbFlags::SharedInfoOnly:
             case CDbFlags::Shared:
                 return CDatabaseReader::getWorkingDbDataFileLocationUrl();
             default:
@@ -531,8 +565,9 @@ namespace BlackCore
             switch (mode)
             {
             case CDbFlags::Shared:
-            case CDbFlags::SharedHeadersOnly:
                 return CDbInfo::entityToSharedName(entity);
+            case CDbFlags::SharedInfoOnly:
+                return CDbInfo::sharedInfoFileName();
             default:
             case CDbFlags::DbReading:
                 return CDbInfo::entityToServiceName(entity);

src/blackcore/db/databasereader.h

@@ -181,8 +181,11 @@ namespace BlackCore
             //! Cache`s number of entities
             virtual int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const = 0;
 
-            //! Info objects available?
+            //! DB info objects available?
-            bool hasInfoObjects() const;
+            bool hasDbInfoObjects() const;
+
+            //! Shared info objects available?
+            bool hasSharedInfoObjects() const;
 
             //! Header of shared file read (for single entity)?
             bool hasSharedFileHeader(const BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -190,26 +193,34 @@ namespace BlackCore
             //! Headers of shared file read (for single entity)?
             bool hasSharedFileHeaders(const BlackMisc::Network::CEntityFlags::Entity entities) const;
 
-            //! Obtain latest object timestamp from info objects
+            //! Obtain latest object timestamp from DB info objects
             //! \sa BlackCore::Db::CInfoDataReader
-            QDateTime getLatestEntityTimestampFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const;
+            QDateTime getLatestEntityTimestampFromDbInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const;
+
+            //! Obtain latest object timestamp from shared info objects
+            //! \sa BlackCore::Db::CInfoDataReader
+            QDateTime getLatestEntityTimestampFromSharedInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const;
 
             //! Header timestamp (last-modified) for shared file
+            //! \deprecated use getLatestEntityTimestampFromSharedInfoObjects
             QDateTime getLatestSharedFileHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
 
-            //! Is the file timestamp neer than cache timestamp?
+            //! Is the file timestamp newer than cache timestamp?
+            //! \deprecated use isSharedInfoNewerThanCacheTimestamp
             bool isSharedHeaderNewerThanCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
 
+            //! Is the shared info timestamp newer than cache timestamp?
+            bool isSharedInfoObjectNewerThanCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
+
             //! Those entities where the timestamp of header is newer than the cache timestamp
             BlackMisc::Network::CEntityFlags::Entity getEntitesWithNewerHeaderTimestamp(BlackMisc::Network::CEntityFlags::Entity entities) const;
 
+            //! Those entities where the timestamp of shared info obejct is newer than the cache timestamp
+            BlackMisc::Network::CEntityFlags::Entity getEntitesWithNewerSharedInfoObject(BlackMisc::Network::CEntityFlags::Entity entities) const;
+
             //! Request header of shared file
             bool requestHeadersOfSharedFiles(BlackMisc::Network::CEntityFlags::Entity entities);
 
-            //! Count from info objects
-            //! \sa BlackCore::Db::CInfoDataReader
-            int getCountFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const;
-
             //! Status message (error message)
             const QString &getStatusMessage() const;
 
@@ -256,9 +267,13 @@ namespace BlackCore
             //! Check if terminated or error, otherwise split into array of objects
             CDatabaseReader::JsonDatastoreResponse setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply);
 
-            //! Info list (latest data timestamp)
+            //! DB Info list (latest data timestamps from DB web service)
             //! \sa BlackCore::Db::CInfoDataReader
-            BlackMisc::Db::CDbInfoList infoList() const;
+            BlackMisc::Db::CDbInfoList getDbInfoObjects() const;
+
+            //! Shared info list (latest data timestamps from DB web service)
+            //! \sa BlackCore::Db::CInfoDataReader
+            BlackMisc::Db::CDbInfoList getSharedInfoObjects() const;
 
             //! Config for given entity
             CDatabaseReaderConfig getConfigForEntity(BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -307,7 +322,7 @@ namespace BlackCore
             //! @}
 
         private:
-            static BlackMisc::Network::CUrl s_workingSharedDbData; //!< one chhosen URL for all DB reader objects
+            static BlackMisc::Network::CUrl s_workingSharedDbData; //!< one choosen URL for all DB reader objects
 
             //! Start reading in own thread (without config/caching)
             //! \remarks can handle DB or shared file reads

src/blackcore/db/databasereaderconfig.cpp

@@ -74,14 +74,14 @@ namespace BlackCore
             return (this->getRetrievalMode().testFlag(CDbFlags::DbReading));
         }
 
-        bool CDatabaseReaderConfig::needsSharedHeader() const
+        bool CDatabaseReaderConfig::needsSharedInfoFile() const
         {
             if (!this->isValid()) { return false; }
             if (!CEntityFlags::anySwiftDbEntity(this->getEntities())) { return false; }
-            return (this->getRetrievalMode().testFlag(CDbFlags::Shared) || this->getRetrievalMode().testFlag(CDbFlags::SharedHeadersOnly));
+            return (this->getRetrievalMode().testFlag(CDbFlags::Shared) || this->getRetrievalMode().testFlag(CDbFlags::SharedInfoOnly));
         }
 
-        bool CDatabaseReaderConfig::needsSharedHeaderLoaded() const
+        bool CDatabaseReaderConfig::needsSharedInfoFileLoaded() const
         {
             if (!this->isValid()) { return false; }
             if (!CEntityFlags::anySwiftDbEntity(this->getEntities())) { return false; }
@@ -162,22 +162,22 @@ namespace BlackCore
             return false;
         }
 
-        bool CDatabaseReaderConfigList::needsSharedHeaders(CEntityFlags::Entity entities) const
+        bool CDatabaseReaderConfigList::needsSharedInfoObjects(CEntityFlags::Entity entities) const
         {
             for (const CDatabaseReaderConfig &config : *this)
             {
                 if (!config.supportsEntities(entities)) { continue; }
-                if (config.needsSharedHeader()) { return true; }
+                if (config.needsSharedInfoFile()) { return true; }
             }
             return false;
         }
 
-        bool CDatabaseReaderConfigList::needsSharedHeadersLoaded(CEntityFlags::Entity entities) const
+        bool CDatabaseReaderConfigList::needsSharedInfoFileLoaded(CEntityFlags::Entity entities) const
         {
             for (const CDatabaseReaderConfig &config : *this)
             {
                 if (!config.supportsEntities(entities)) { continue; }
-                if (config.needsSharedHeaderLoaded()) { return true; }
+                if (config.needsSharedInfoFileLoaded()) { return true; }
             }
             return false;
         }

src/blackcore/db/databasereaderconfig.h

@@ -66,10 +66,10 @@ namespace BlackCore
             bool possiblyReadsFromSwiftDb() const;
 
             //! Needs the shared header
-            bool needsSharedHeader() const;
+            bool needsSharedInfoFile() const;
 
             //! Needs the shared header loaded before it can be continued
-            bool needsSharedHeaderLoaded() const;
+            bool needsSharedInfoFileLoaded() const;
 
             //! Will write to swift DB
             bool possiblyWritesToSwiftDb() const;
@@ -122,10 +122,10 @@ namespace BlackCore
             bool possiblyWritesToSwiftDb() const;
 
             //! Needs any shared header
-            bool needsSharedHeaders(BlackMisc::Network::CEntityFlags::Entity entities) const;
+            bool needsSharedInfoObjects(BlackMisc::Network::CEntityFlags::Entity entities) const;
 
             //! Needs any shared header loaded before continued
-            bool needsSharedHeadersLoaded(BlackMisc::Network::CEntityFlags::Entity entities) const;
+            bool needsSharedInfoFileLoaded(BlackMisc::Network::CEntityFlags::Entity entities) const;
 
             //! Entities which will use cache or DB, so no canceled or ignored ones
             BlackMisc::Network::CEntityFlags::Entity getEntitesCachedOrReadFromDB() const;

src/blackcore/db/infodatareader.cpp

@@ -26,17 +26,19 @@ namespace BlackCore
 {
     namespace Db
     {
-        CInfoDataReader::CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config) :
+        CInfoDataReader::CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config, CDbFlags::DataRetrievalModeFlag mode) :
-            CDatabaseReader(owner, config, "CInfoDataReader")
+            CDatabaseReader(owner, config, "CInfoDataReader"), m_mode(mode)
-        { }
+        {
+            Q_ASSERT_X(mode == CDbFlags::DbReading || mode == CDbFlags::Shared, Q_FUNC_INFO, "Wrong mode");
+        }
 
-        CDbInfoList CInfoDataReader::getDbInfoObjects() const
+        CDbInfoList CInfoDataReader::getInfoObjects() const
         {
             QReadLocker l(&m_lockInfoObjects);
             return m_infoObjects;
         }
 
-        int CInfoDataReader::getDbInfoObjectCount() const
+        int CInfoDataReader::getInfoObjectCount() const
         {
             QReadLocker l(&m_lockInfoObjects);
             return m_infoObjects.size();
@@ -44,7 +46,7 @@ namespace BlackCore
 
         bool CInfoDataReader::areAllDataRead() const
         {
-            return getDbInfoObjectCount() > 4;
+            return getInfoObjectCount() > 4;
         }
 
         void CInfoDataReader::synchronizeCaches(CEntityFlags::Entity entities)
@@ -93,15 +95,15 @@ namespace BlackCore
 
         void CInfoDataReader::read()
         {
-            const CUrl url(getDbInfoObjectsUrl());
+            const CUrl url(this->getInfoObjectsUrl());
             if (!url.isEmpty())
             {
                 sApp->getFromNetwork(url, { this, &CInfoDataReader::ps_parseInfoObjectsData});
-                emit dataRead(CEntityFlags::InfoObjectEntity, CEntityFlags::StartRead, 0);
+                emit dataRead(this->getEntityForMode(), CEntityFlags::StartRead, 0);
             }
             else
             {
-                CLogMessage(this).error("No URL for %1") << CEntityFlags::flagToString(CEntityFlags::InfoObjectEntity);
+                CLogMessage(this).error("No URL for '%1'") << CEntityFlags::flagToString(this->getEntityForMode());
             }
         }
 
@@ -112,11 +114,11 @@ namespace BlackCore
             QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
             if (this->isShuttingDown()) { return; }
 
-            CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
+            const CDatabaseReader::JsonDatastoreResponse res = this->setStatusAndTransformReplyIntoDatastoreResponse(nwReply.data());
             if (res.hasErrorMessage())
             {
                 CLogMessage::preformatted(res.lastWarningOrAbove());
-                emit dataRead(CEntityFlags::InfoObjectEntity, CEntityFlags::ReadFailed, 0);
+                emit dataRead(this->getEntityForMode(), CEntityFlags::ReadFailed, 0);
                 return;
             }
 
@@ -130,7 +132,7 @@ namespace BlackCore
                 this->m_infoObjects = infoObjects;
             }
 
-            this->emitAndLogDataRead(CEntityFlags::InfoObjectEntity, n, res);
+            this->emitAndLogDataRead(this->getEntityForMode(), n, res);
         }
 
         CUrl CInfoDataReader::getDbInfoObjectsUrl() const
@@ -138,9 +140,34 @@ namespace BlackCore
             return getBaseUrl(CDbFlags::DbReading).withAppendedPath("jsondbinfo.php");
         }
 
+        CUrl CInfoDataReader::getSharedInfoObjectsUrl() const
+        {
+            return getBaseUrl(CDbFlags::Shared).withAppendedPath(CDbInfo::sharedInfoFileName());
+        }
+
+        CEntityFlags::EntityFlag CInfoDataReader::getEntityForMode() const
+        {
+            if (this->m_mode == CDbFlags::DbReading) return CEntityFlags::DbInfoObjectEntity;
+            if (this->m_mode == CDbFlags::Shared) return CEntityFlags::SharedInfoObjectEntity;
+            qFatal("Wrong mode");
+            return CEntityFlags::NoEntity;
+        }
+
+        CUrl CInfoDataReader::getInfoObjectsUrl() const
+        {
+            switch (m_mode)
+            {
+            case CDbFlags::DbReading: return getDbInfoObjectsUrl();
+            case CDbFlags::Shared: return getSharedInfoObjectsUrl();
+            default:
+                qFatal("Wrong mode");
+            }
+            return CUrl();
+        }
+
         CEntityFlags::Entity CInfoDataReader::getSupportedEntities() const
         {
-            return CEntityFlags::InfoObjectEntity;
+            return this->getEntityForMode();
         }
     } // namespace
 } // namespace

src/blackcore/db/infodatareader.h

@@ -24,33 +24,33 @@ namespace BlackCore
 {
     namespace Db
     {
-        //! Read information about data from Database
+        //! Read information about data from Database or shared file
         class BLACKCORE_EXPORT CInfoDataReader : public CDatabaseReader
         {
             Q_OBJECT
 
         public:
             //! Constructor
-            explicit CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config);
+            explicit CInfoDataReader(QObject *owner, const CDatabaseReaderConfigList &config, BlackMisc::Db::CDbFlags::DataRetrievalModeFlag mode);
 
-            //! Get info list
+            //! Get info list (either shared or from DB)
             //! \threadsafe
-            BlackMisc::Db::CDbInfoList getDbInfoObjects() const;
+            BlackMisc::Db::CDbInfoList getInfoObjects() const;
 
-            //! Get info list size
+            //! Get info list size (either shared or from DB)
             //! \threadsafe
-            int getDbInfoObjectCount() const;
+            int getInfoObjectCount() const;
 
             //! All data read?
             //! \threadsafe
             bool areAllDataRead() const;
 
-            //! URL info objects web service
-            BlackMisc::Network::CUrl getDbInfoObjectsUrl() const;
-
             //! Allow to call directly, special for info objects reader
             void read();
 
+            //! URL depending on mode
+            BlackMisc::Network::CUrl getInfoObjectsUrl() const;
+
             // cache handling for base class: no cache handling here in that case
             virtual BlackMisc::Network::CEntityFlags::Entity getSupportedEntities() const override;
             virtual QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const override;
@@ -69,6 +69,16 @@ namespace BlackCore
             void ps_parseInfoObjectsData(QNetworkReply *nwReply);
 
         private:
+            //! URL info objects web service
+            BlackMisc::Network::CUrl getDbInfoObjectsUrl() const;
+
+            //! URL shared info objects
+            BlackMisc::Network::CUrl getSharedInfoObjectsUrl() const;
+
+            //! Entity for mode
+            BlackMisc::Network::CEntityFlags::EntityFlag getEntityForMode() const;
+
+            BlackMisc::Db::CDbFlags::DataRetrievalModeFlag m_mode; //!< shared or DB web service?
             BlackMisc::Db::CDbInfoList m_infoObjects;
             BlackMisc::Network::CUrl   m_urlInfoObjects;
             mutable QReadWriteLock     m_lockInfoObjects;