refs #787, support for Header / shared files in database reader / web services

* split JsonDatastoreResponse in subclass HeaderResponse
* support for request newtwork request duration ("started") in application
* removed ps_setupChanged because it was unused

samples/blackmisc/samplesperformance.cpp

@@ -360,7 +360,8 @@ namespace BlackSample
         Q_ASSERT_X(!data.isEmpty(), Q_FUNC_INFO, "Model file empty");
 
         // DB format, all models denormalized in DB JSON format
-        CDatabaseReader::JsonDatastoreResponse response = CDatabaseReader::stringToDatastoreResponse(data);
+        CDatabaseReader::JsonDatastoreResponse response;
+        CDatabaseReader::stringToDatastoreResponse(data, response);
         QTime timer;
         timer.start();
         const CAircraftModelList dbModels = CAircraftModelList::fromDatabaseJson(response);

src/blackcore/application.cpp

@@ -483,12 +483,12 @@ namespace BlackCore
         });
     }
 
-    QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const BlackMisc::CSlot<void (QNetworkReply *)> &callback)
+    QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const CSlot<void (QNetworkReply *)> &callback)
     {
         return httpRequestImpl(url.toNetworkRequest(), callback, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.head(request); });
     }
 
-    QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const BlackMisc::CSlot<void (QNetworkReply *)> &callback)
+    QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const CSlot<void (QNetworkReply *)> &callback)
     {
         return httpRequestImpl(request, callback, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.head(request); });
     }
@@ -1021,7 +1021,7 @@ namespace BlackCore
         }
 
         CUrl serverUrl;
-        serverUrl = getGlobalSetup().getCrashreportServerUrl();
+        serverUrl = getGlobalSetup().getCrashReportServerUrl();
         std::map<std::string, std::string> annotations;
 
         // Caliper (mini-breakpad-server) annotations
@@ -1063,6 +1063,7 @@ namespace BlackCore
         CNetworkUtils::ignoreSslVerification(r);
         CNetworkUtils::setSwiftUserAgent(r);
         QNetworkReply *reply = method(this->m_accessManager, r);
+        reply->setProperty("started", QVariant(QDateTime::currentMSecsSinceEpoch()));
         if (callback)
         {
             connect(reply, &QNetworkReply::finished, callback.object(), [ = ] { callback(reply); }, Qt::QueuedConnection);

src/blackcore/db/databasereader.cpp

@@ -31,6 +31,7 @@ using namespace BlackMisc;
 using namespace BlackMisc::Db;
 using namespace BlackMisc::Network;
 using namespace BlackCore;
+using namespace BlackCore::Data;
 
 namespace BlackCore
 {
@@ -130,41 +131,74 @@ namespace BlackCore
 
         CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const
         {
-            this->threadAssertCheck();
-
             JsonDatastoreResponse datastoreResponse;
-            if (this->isAbandoned())
+            const bool ok = this->setHeaderInfoPart(datastoreResponse, nwReply);
-            {
+            if (ok)
-                nwReply->abort();
-                datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Terminated data parsing process"));
-                return datastoreResponse; // stop, terminate straight away, ending thread
-            }
-
-            if (nwReply->error() == QNetworkReply::NoError)
             {
                 const QString dataFileData = nwReply->readAll().trimmed();
                 nwReply->close(); // close asap
                 if (dataFileData.isEmpty())
                 {
                     datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data"));
-                    datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
+                    datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc());
                 }
                 else
                 {
-                    datastoreResponse = CDatabaseReader::stringToDatastoreResponse(dataFileData);
+                    CDatabaseReader::stringToDatastoreResponse(dataFileData, datastoreResponse);
                 }
             }
-            else
+            return datastoreResponse;
-            {
+        }
 
+        CDatabaseReader::HeaderResponse CDatabaseReader::transformReplyIntoHeaderResponse(QNetworkReply *nwReply) const
+        {
+            HeaderResponse headerResponse;
+            const bool success = this->setHeaderInfoPart(headerResponse, nwReply);
+            Q_UNUSED(success);
+            nwReply->close();
+            return headerResponse;
+        }
+
+        bool CDatabaseReader::setHeaderInfoPart(CDatabaseReader::HeaderResponse &headerResponse, QNetworkReply *nwReply) const
+        {
+            Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing reply");
+            this->threadAssertCheck();
+            if (this->isAbandoned())
+            {
+                nwReply->abort();
+                headerResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Terminated data parsing process"));
+                return false; // stop, terminate straight away, ending thread
+            }
+
+            headerResponse.setUrl(nwReply->url());
+            const QVariant started = nwReply->property("started");
+            if (started.isValid() && started.canConvert<qint64>())
+            {
+                const qint64 now = QDateTime::currentMSecsSinceEpoch();
+                const qint64 start = started.value<qint64>();
+                headerResponse.setLoadTimeMs(now - start);
+            }
+
+            const QDateTime lastModified = nwReply->header(QNetworkRequest::LastModifiedHeader).toDateTime();
+            const qulonglong size = static_cast<qlonglong>(nwReply->header(QNetworkRequest::ContentLengthHeader).toULongLong());
+            headerResponse.setUpdateTimestamp(lastModified);
+            headerResponse.setContentLengthHeader(size);
+
+            if (nwReply->error() == QNetworkReply::NoError)
+            {
+                // do not close because of obtaining data
+                return true;
+            }
+            else
+            {
                 // no valid response
                 const QString error(nwReply->errorString());
                 const QString url(nwReply->url().toString());
                 nwReply->abort();
-                datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError,
+                headerResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError,
                                           QString("Reading data failed: " + error + " " + url)));
+                return false;
             }
-            return datastoreResponse;
         }
 
         CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply)
@@ -191,11 +225,50 @@ namespace BlackCore
             static const QDateTime e;
             const CDbInfoList il(infoList());
             if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; }
-            CDbInfo info = il.findFirstByEntityOrDefault(entity);
+
+            // for some entities there can be more than one entry because of the
+            // raw tables (see DB view last updates)
+            const CDbInfo info = il.findFirstByEntityOrDefault(entity);
             if (!info.isValid()) { return e; }
             return info.getUtcTimestamp();
         }
 
+        QDateTime CDatabaseReader::getSharedFileTimestamp(CEntityFlags::Entity entity) const
+        {
+            const int key = static_cast<int>(entity);
+            if (m_sharedFileResponses.contains(key))
+            {
+                return this->m_sharedFileResponses[key].getUpdateTimestamp();
+            }
+            else
+            {
+                return QDateTime();
+            }
+        }
+
+        bool CDatabaseReader::requestHeadersOfSharedFiles(const CEntityFlags::Entity &entities)
+        {
+            CEntityFlags::Entity allEntities(entities);
+            CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities);
+            CUrl urlDbData = CGlobalSetup::buildDbDataDirectory(getWorkingSharedUrl());
+
+            int c = 0;
+            while (currentEntity != CEntityFlags::NoEntity)
+            {
+                const QString fileName = CDbInfo::entityToSharedFileName(currentEntity);
+                Q_ASSERT_X(!fileName.isEmpty(), Q_FUNC_INFO, "No file name for entity");
+                CUrl url = urlDbData;
+                url.appendPath(fileName);
+
+                const QString entityString = CEntityFlags::flagToString(currentEntity);
+                CLogMessage(this).info("Triggered read of header for shared file of %1") << entityString;
+                const QNetworkReply *reply = sApp->headerFromNetwork(url, { this, &CDatabaseReader::receivedSharedFileHeader });
+                if (reply) { c++; }
+                currentEntity = CEntityFlags::iterateDbEntities(allEntities);
+            }
+            return c > 0;
+        }
+
         int CDatabaseReader::getCountFromInfoObjects(CEntityFlags::Entity entity) const
         {
             static const QDateTime e;
@@ -240,6 +313,22 @@ namespace BlackCore
             return oldS != currentS;
         }
 
+        void CDatabaseReader::receivedSharedFileHeader(QNetworkReply *nwReplyPtr)
+        {
+            // wrap pointer, make sure any exit cleans up reply
+            // required to use delete later as object is created in a different thread
+            QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> nwReply(nwReplyPtr);
+            if (this->isAbandoned()) { return; }
+
+            const HeaderResponse headerResponse = this->transformReplyIntoHeaderResponse(nwReply.data());
+            const QString fileName = nwReply->url().fileName();
+            const CEntityFlags::Entity entity = CEntityFlags::singleEntityByName(fileName);
+            const int key = static_cast<int>(entity);
+            this->m_sharedFileResponses[key] = headerResponse;
+
+            emit this->sharedFileHeaderRead(entity, fileName, !headerResponse.hasWarningOrAboveMessage());
+        }
+
         bool CDatabaseReader::hasReceivedOkReply() const
         {
             QReadLocker rl(&this->m_statusLock);
@@ -324,14 +413,13 @@ namespace BlackCore
             return CNetworkUtils::canConnect(url);
         }
 
-        CDatabaseReader::JsonDatastoreResponse CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent)
+        void CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent, JsonDatastoreResponse &datastoreResponse)
         {
-            CDatabaseReader::JsonDatastoreResponse datastoreResponse;
             if (jsonContent.isEmpty())
             {
                 datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty string, no data"));
-                datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
+                datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc());
-                return datastoreResponse;
+                return;
             }
 
             const QJsonDocument jsonResponse = QJsonDocument::fromJson(jsonContent.toUtf8());
@@ -339,17 +427,16 @@ namespace BlackCore
             {
                 // directly an array, no further info
                 datastoreResponse.setJsonArray(jsonResponse.array());
-                datastoreResponse.m_updated = QDateTime::currentDateTimeUtc();
+                datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc());
             }
             else
             {
                 const QJsonObject responseObject(jsonResponse.object());
                 datastoreResponse.setJsonArray(responseObject["data"].toArray());
                 const QString ts(responseObject["latest"].toString());
-                datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts);
+                datastoreResponse.setUpdateTimestamp(ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts));
-                datastoreResponse.m_restricted = responseObject["restricted"].toBool();
+                datastoreResponse.setRestricted(responseObject["restricted"].toBool());
             }
-            return datastoreResponse;
         }
 
         void CDatabaseReader::JsonDatastoreResponse::setJsonArray(const QJsonArray &value)

src/blackcore/db/databasereader.h

@@ -24,6 +24,7 @@
 
 #include <QDateTime>
 #include <QJsonArray>
+#include <QMap>
 #include <QObject>
 #include <QReadWriteLock>
 #include <QString>
@@ -44,21 +45,17 @@ namespace BlackCore
             Q_OBJECT
 
         public:
-            //!  Response from our database
+            //! Header response part
-            struct JsonDatastoreResponse
+            struct HeaderResponse
             {
-                QJsonArray m_jsonArray;              //!< JSON array data
+            private:
-                QDateTime  m_updated;                //!< when was the latest updated?
+                QDateTime  m_updated;                  //!< when was the latest update?
-                int        m_arraySize  = -1;        //!< size of array, if applicable (copied to member for debugging purposes)
-                bool       m_restricted = false;     //!< restricted reponse, only changed data
                 BlackMisc::CStatusMessage m_message;   //!< last error or warning
+                BlackMisc::Network::CUrl  m_url;       //!< loaded url
+                qulonglong  m_contentLengthHeader = 0; //!< content length
+                qint64     m_loadTimeMs = -1;          //!< how long did it take to load
 
-                //! Any data?
+            public:
-                bool isEmpty() const { return m_jsonArray.isEmpty(); }
-
-                //! Number of elements
-                int size() const { return m_jsonArray.size(); }
-
                 //! Any timestamp?
                 bool hasTimestamp() const { return m_updated.isValid(); }
 
@@ -68,8 +65,17 @@ namespace BlackCore
                 //! Is response newer?
                 bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; }
 
-                //! Incremental data
+                //! Get the update timestamp
-                bool isRestricted() const { return m_restricted; }
+                const QDateTime &getUpdateTimestamp() const { return m_updated; }
+
+                //! Set update timestamp, default normally "last-modified"
+                void setUpdateTimestamp(const QDateTime &updated) { m_updated = updated; }
+
+                //! Header content length
+                qulonglong getContentLengthHeader() const { return m_contentLengthHeader; }
+
+                //! Set the content length
+                void setContentLengthHeader(qulonglong size) { m_contentLengthHeader = size; }
 
                 //! Error message?
                 bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; }
@@ -83,6 +89,40 @@ namespace BlackCore
                 //! Set the error/warning message
                 void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; }
 
+                //! URL loaded
+                const BlackMisc::Network::CUrl &getUrl() const { return m_url; }
+
+                //! Set the loaded URL
+                void setUrl(const BlackMisc::Network::CUrl &url) { m_url = url; }
+
+                //! Load time in ms (from request to response)
+                qint64 getLoadTimeMs() const { return m_loadTimeMs; }
+
+                //! Set the load time (delta start -> response received)
+                void setLoadTimeMs(qint64 deltaTime) { m_loadTimeMs = deltaTime; }
+            };
+
+            //!  Response from our database (depneding on JSON DB backend generates)
+            struct JsonDatastoreResponse : public HeaderResponse
+            {
+            private:
+                QJsonArray m_jsonArray;              //!< JSON array data
+                int        m_arraySize  = -1;        //!< size of array, if applicable (copied to member for debugging purposes)
+                bool       m_restricted = false;     //!< restricted reponse, only changed data
+
+            public:
+                //! Any data?
+                bool isEmpty() const { return m_jsonArray.isEmpty(); }
+
+                //! Number of elements
+                int size() const { return m_jsonArray.size(); }
+
+                //! Incremental data, restricted by query?
+                bool isRestricted() const { return m_restricted; }
+
+                //! Incremental data, restricted by query
+                void setRestricted(bool restricted) { m_restricted = restricted; }
+
                 //! Get the JSON array
                 QJsonArray getJsonArray() const { return m_jsonArray; }
 
@@ -125,6 +165,12 @@ namespace BlackCore
             //! \sa BlackCore::Db::CInfoDataReader
             QDateTime getLatestEntityTimestampFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const;
 
+            //! Timestamp of shared file for entiry
+            QDateTime getSharedFileTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
+
+            //! Request header of shared file
+            bool requestHeadersOfSharedFiles(const BlackMisc::Network::CEntityFlags::Entity &entities);
+
             //! Count from info objects
             //! \sa BlackCore::Db::CInfoDataReader
             int getCountFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -141,18 +187,23 @@ namespace BlackCore
             //! Name of parameter for latest id
             static const QString &parameterLatestId();
 
-            //! sift DB server reachable?
+            //! swift DB server reachable?
             static bool canPingSwiftServer();
 
-            //! Transform JSON data to response struct
+            //! Transform JSON data to response struct data
-            static JsonDatastoreResponse stringToDatastoreResponse(const QString &jsonContent);
+            //! \private used also for samples
+            static void stringToDatastoreResponse(const QString &jsonContent, CDatabaseReader::JsonDatastoreResponse &datastoreResponse);
 
         signals:
             //! Combined read signal
             void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
 
+            //! Header of shared file read
+            void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success);
+
         protected:
             CDatabaseReaderConfigList   m_config;                      //!< DB reder configuration
+            QMap<int, HeaderResponse>   m_sharedFileResponses;         //!< file responses of the shared files
             QString                     m_statusMessage;               //!< Returned status message from watchdog
             QNetworkReply::NetworkError m_1stReplyStatus = QNetworkReply::UnknownServerError; //!< Successful connection?
             bool                        m_1stReplyReceived = false;    //!< Successful connection? Does not mean data / authorizations are correct
@@ -202,9 +253,18 @@ namespace BlackCore
             //! @}
 
         private:
+            //! Received a reply of a header for a shared file
+            void receivedSharedFileHeader(QNetworkReply *nwReplyPtr);
+
             //! Check if terminated or error, otherwise split into array of objects
             JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const;
 
+            //! Check if terminated or error, otherwise set header information
+            HeaderResponse transformReplyIntoHeaderResponse(QNetworkReply *nwReply) const;
+
+            //! Set the header part
+            bool setHeaderInfoPart(HeaderResponse &headerResponse, QNetworkReply *nwReply) const;
+
             //! Feedback about connection status
             //! \threadsafe
             void setReplyStatus(QNetworkReply::NetworkError status, const QString &message = "");

src/blackcore/webdataservices.cpp

@@ -332,6 +332,27 @@ namespace BlackCore
         }
     }
 
+    QDateTime CWebDataServices::getSharedFileTimestamp(CEntityFlags::Entity entity) const
+    {
+        const CDatabaseReader *reader = this->getDbReader(entity);
+        if (reader)
+        {
+            return reader->getSharedFileTimestamp(entity);
+        }
+        else
+        {
+            return QDateTime();
+        }
+    }
+
+    bool CWebDataServices::requestHeaderOfSharedFile(CEntityFlags::Entity entity)
+    {
+        Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity");
+        CDatabaseReader *reader = this->getDbReader(entity);
+        if (!reader) { return false; }
+        return reader->requestHeadersOfSharedFiles(entity);
+    }
+
     int CWebDataServices::getCacheCount(CEntityFlags::Entity entity) const
     {
         Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity");
@@ -618,7 +639,7 @@ namespace BlackCore
         if (this->m_databaseWriter)       { this->m_databaseWriter->gracefulShutdown(); }
     }
 
-    CEntityFlags::Entity CWebDataServices::allDbEntiiesUsed() const
+    CEntityFlags::Entity CWebDataServices::allDbEntiiesForUsedReaders() const
     {
         // obtain entities from real readers (means when reader is really used)
         CEntityFlags::Entity entities = CEntityFlags::NoEntity;
@@ -671,9 +692,13 @@ namespace BlackCore
             {
                 this->m_infoDataReader = new CInfoDataReader(this, dbReaderConfig);
                 c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
-                Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO info object signals");
+                Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed");
+
+                // relay signal
                 c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead);
-                Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed info data");
+                Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed");
+
+                // start reading
                 this->m_infoDataReader->start(QThread::LowPriority);
                 QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); });
             }
@@ -711,7 +736,7 @@ namespace BlackCore
         if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader))
         {
             this->m_vatsimDataFileReader = new CVatsimDataFileReader(this);
-            c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_dataFileRead);
+            c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_vatsimDataFileRead);
             Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals");
             c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead);
             Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file");
@@ -738,9 +763,11 @@ namespace BlackCore
         {
             this->m_icaoDataReader = new CIcaoDataReader(this, dbReaderConfig);
             c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals");
             c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead);
             Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals");
+            c = connect(this->m_icaoDataReader, &CIcaoDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead);
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals");
             this->m_icaoDataReader->start(QThread::LowPriority);
         }
 
@@ -749,9 +776,11 @@ namespace BlackCore
         {
             this->m_modelDataReader = new CModelDataReader(this, dbReaderConfig);
             c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals");
             c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed models");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals");
+            c = connect(this->m_modelDataReader, &CModelDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead);
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals");
             this->m_modelDataReader->start(QThread::LowPriority);
         }
 
@@ -760,11 +789,17 @@ namespace BlackCore
         {
             this->m_airportDataReader = new CAirportDataReader(this, dbReaderConfig);
             c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "Airport reader signals");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals");
             c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::dataRead);
-            Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed for airports");
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals");
+            c = connect(this->m_airportDataReader, &CAirportDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead);
+            Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals");
             this->m_airportDataReader->start(QThread::LowPriority);
         }
+
+        // Trigger Shared file headers loading
+        //! \todo refine, check if really needed to load the headers here
+        QTimer::singleShot(0, [this]() { this->triggerLoadingOfSharedFilesHeaders(); });
     }
 
     CDatabaseReader *CWebDataServices::getDbReader(CEntityFlags::Entity entity) const
@@ -772,7 +807,7 @@ namespace BlackCore
         Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity");
         Q_ASSERT_X(CEntityFlags::anySwiftDbEntity(entity), Q_FUNC_INFO, "No swift DB entity");
 
-        CWebReaderFlags::WebReader wr = CWebReaderFlags::entityToReader(entity);
+        const CWebReaderFlags::WebReader wr = CWebReaderFlags::entityToReader(entity);
         switch (wr)
         {
         case CWebReaderFlags::IcaoDataReader: return this->m_icaoDataReader;
@@ -801,7 +836,7 @@ namespace BlackCore
         CLogMessage(this).info("Read %1 METARs") << metars.size();
     }
 
-    void CWebDataServices::ps_dataFileRead(int lines)
+    void CWebDataServices::ps_vatsimDataFileRead(int lines)
     {
         CLogMessage(this).info("Read VATSIM data file, %1 lines") << lines;
     }
@@ -829,18 +864,13 @@ namespace BlackCore
         }
 
         this->m_swiftDbEntitiesRead |= entity;
-        const int allUsedEntities = static_cast<int>(this->allDbEntiiesUsed());
+        const int allUsedEntities = static_cast<int>(this->allDbEntiiesForUsedReaders());
         if (((static_cast<int>(this->m_swiftDbEntitiesRead)) & allUsedEntities) == allUsedEntities)
         {
             emit allSwiftDbDataRead();
         }
     }
 
-    void CWebDataServices::ps_setupChanged()
-    {
-        // void
-    }
-
     void CWebDataServices::readDeferredInBackground(CEntityFlags::Entity entities, int delayMs)
     {
         if (entities == CEntityFlags::NoEntity) { return; }
@@ -959,7 +989,7 @@ namespace BlackCore
     bool CWebDataServices::readDbDataFromDisk(const QString &dir, bool inBackground)
     {
         if (dir.isEmpty()) { return false; }
-        QDir directory(dir);
+        const QDir directory(dir);
         if (!directory.exists()) { return false; }
 
         bool s = false;
@@ -977,4 +1007,37 @@ namespace BlackCore
         }
         return s;
     }
+
+    bool CWebDataServices::triggerLoadingOfSharedFilesHeaders(CEntityFlags::Entity requestedEntities)
+    {
+        CEntityFlags::Entity triggeredEntities = CEntityFlags::NoEntity;
+        if (this->m_modelDataReader)
+        {
+            const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::ModelReader);
+            if (entities != CEntityFlags::NoEntity)
+            {
+                triggeredEntities |= entities;
+                this->m_modelDataReader->requestHeadersOfSharedFiles(entities);
+            }
+        }
+        if (this->m_icaoDataReader)
+        {
+            const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::IcaoDataReader);
+            if (entities != CEntityFlags::NoEntity)
+            {
+                triggeredEntities |= entities;
+                this->m_icaoDataReader->requestHeadersOfSharedFiles(entities);
+            }
+        }
+        if (this->m_airportDataReader)
+        {
+            const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::AirportReader);
+            if (entities != CEntityFlags::NoEntity)
+            {
+                triggeredEntities |= entities;
+                this->m_airportDataReader->requestHeadersOfSharedFiles(entities);
+            }
+        }
+        return triggeredEntities != CEntityFlags::NoEntity;
+    }
 } // ns

src/blackcore/webdataservices.h

@@ -116,8 +116,8 @@ namespace BlackCore
         //! Reader flags
         CWebReaderFlags::WebReader getReaderFlags() const { return m_readers; }
 
-        //! All DB entities for those used and not ignored
+        //! All DB entities for those readers used and not ignored
-        BlackMisc::Network::CEntityFlags::Entity allDbEntiiesUsed() const;
+        BlackMisc::Network::CEntityFlags::Entity allDbEntiiesForUsedReaders() const;
 
         //! FSD servers
         //! \threadsafe
@@ -317,6 +317,10 @@ namespace BlackCore
         //! Trigger reload from DB, only loads the DB data and bypasses the caches checks and info objects
         BlackMisc::Network::CEntityFlags::Entity triggerReloadFromDb(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime());
 
+        //! Trigger loading of the HTTP headers for the shared files
+        //! \note allows to obtain the timestamps
+        bool triggerLoadingOfSharedFilesHeaders(BlackMisc::Network::CEntityFlags::Entity requestedEntities = BlackMisc::Network::CEntityFlags::AllDbEntities);
+
         //! Corresponding cache timestamp if applicable
         //! \threadsafe
         QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -325,6 +329,12 @@ namespace BlackCore
         //! \threadsafe
         QDateTime getDbLatestEntityTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
 
+        //! Corresponding shared file timestamp
+        QDateTime getSharedFileTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const;
+
+        //! Request (updated) HTTP header for shared file of entity
+        bool requestHeaderOfSharedFile(BlackMisc::Network::CEntityFlags::Entity entity);
+
         //! Cache count for entity
         //! \threadsafe
         int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const;
@@ -358,6 +368,9 @@ namespace BlackCore
         //! All swift DB data have been read
         void allSwiftDbDataRead();
 
+        //! Header of shared file read
+        void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success);
+
     public slots:
         //! Call CWebDataServices::readInBackground by single shot
         void readDeferredInBackground(BlackMisc::Network::CEntityFlags::Entity entities, int delayMs);
@@ -373,14 +386,11 @@ namespace BlackCore
         void ps_receivedMetars(const BlackMisc::Weather::CMetarList &metars);
 
         //! Data file has been read
-        void ps_dataFileRead(int lines);
+        void ps_vatsimDataFileRead(int lines);
 
         //! Read finished from reader
         void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number);
 
-        //! Setup changed
-        void ps_setupChanged();
-
     private:
         //! Init the readers
         void initReaders(CWebReaderFlags::WebReader flags);