From ff394f4785e1d882af76f3b1c006df25e26cedbc Mon Sep 17 00:00:00 2001 From: Klaus Basan Date: Thu, 3 Nov 2016 02:37:17 +0100 Subject: [PATCH] refs #787, support for Header / shared files in database reader / web services * split JsonDatastoreResponse in subclass HeaderResponse * support for request newtwork request duration ("started") in application * removed ps_setupChanged because it was unused --- samples/blackmisc/samplesperformance.cpp | 3 +- src/blackcore/application.cpp | 7 +- src/blackcore/db/databasereader.cpp | 137 ++++++++++++++++++----- src/blackcore/db/databasereader.h | 98 ++++++++++++---- src/blackcore/webdataservices.cpp | 99 +++++++++++++--- src/blackcore/webdataservices.h | 22 +++- 6 files changed, 294 insertions(+), 72 deletions(-) diff --git a/samples/blackmisc/samplesperformance.cpp b/samples/blackmisc/samplesperformance.cpp index f38718c09..75203b3bd 100644 --- a/samples/blackmisc/samplesperformance.cpp +++ b/samples/blackmisc/samplesperformance.cpp @@ -360,7 +360,8 @@ namespace BlackSample Q_ASSERT_X(!data.isEmpty(), Q_FUNC_INFO, "Model file empty"); // DB format, all models denormalized in DB JSON format - CDatabaseReader::JsonDatastoreResponse response = CDatabaseReader::stringToDatastoreResponse(data); + CDatabaseReader::JsonDatastoreResponse response; + CDatabaseReader::stringToDatastoreResponse(data, response); QTime timer; timer.start(); const CAircraftModelList dbModels = CAircraftModelList::fromDatabaseJson(response); diff --git a/src/blackcore/application.cpp b/src/blackcore/application.cpp index 751717019..e0ff2fadb 100644 --- a/src/blackcore/application.cpp +++ b/src/blackcore/application.cpp @@ -483,12 +483,12 @@ namespace BlackCore }); } - QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const BlackMisc::CSlot &callback) + QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const CSlot &callback) { return httpRequestImpl(url.toNetworkRequest(), callback, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.head(request); }); } - QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const BlackMisc::CSlot &callback) + QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const CSlot &callback) { return httpRequestImpl(request, callback, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.head(request); }); } @@ -1021,7 +1021,7 @@ namespace BlackCore } CUrl serverUrl; - serverUrl = getGlobalSetup().getCrashreportServerUrl(); + serverUrl = getGlobalSetup().getCrashReportServerUrl(); std::map annotations; // Caliper (mini-breakpad-server) annotations @@ -1063,6 +1063,7 @@ namespace BlackCore CNetworkUtils::ignoreSslVerification(r); CNetworkUtils::setSwiftUserAgent(r); QNetworkReply *reply = method(this->m_accessManager, r); + reply->setProperty("started", QVariant(QDateTime::currentMSecsSinceEpoch())); if (callback) { connect(reply, &QNetworkReply::finished, callback.object(), [ = ] { callback(reply); }, Qt::QueuedConnection); diff --git a/src/blackcore/db/databasereader.cpp b/src/blackcore/db/databasereader.cpp index fd2b80c23..f8712fe92 100644 --- a/src/blackcore/db/databasereader.cpp +++ b/src/blackcore/db/databasereader.cpp @@ -31,6 +31,7 @@ using namespace BlackMisc; using namespace BlackMisc::Db; using namespace BlackMisc::Network; using namespace BlackCore; +using namespace BlackCore::Data; namespace BlackCore { @@ -130,41 +131,74 @@ namespace BlackCore CDatabaseReader::JsonDatastoreResponse CDatabaseReader::transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const { - this->threadAssertCheck(); - JsonDatastoreResponse datastoreResponse; - if (this->isAbandoned()) - { - nwReply->abort(); - datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Terminated data parsing process")); - return datastoreResponse; // stop, terminate straight away, ending thread - } - - if (nwReply->error() == QNetworkReply::NoError) + const bool ok = this->setHeaderInfoPart(datastoreResponse, nwReply); + if (ok) { const QString dataFileData = nwReply->readAll().trimmed(); nwReply->close(); // close asap if (dataFileData.isEmpty()) { datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Empty response, no data")); - datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); + datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc()); } else { - datastoreResponse = CDatabaseReader::stringToDatastoreResponse(dataFileData); + CDatabaseReader::stringToDatastoreResponse(dataFileData, datastoreResponse); } } + return datastoreResponse; + } + + CDatabaseReader::HeaderResponse CDatabaseReader::transformReplyIntoHeaderResponse(QNetworkReply *nwReply) const + { + HeaderResponse headerResponse; + const bool success = this->setHeaderInfoPart(headerResponse, nwReply); + Q_UNUSED(success); + nwReply->close(); + return headerResponse; + } + + bool CDatabaseReader::setHeaderInfoPart(CDatabaseReader::HeaderResponse &headerResponse, QNetworkReply *nwReply) const + { + Q_ASSERT_X(nwReply, Q_FUNC_INFO, "Missing reply"); + this->threadAssertCheck(); + if (this->isAbandoned()) + { + nwReply->abort(); + headerResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, "Terminated data parsing process")); + return false; // stop, terminate straight away, ending thread + } + + headerResponse.setUrl(nwReply->url()); + const QVariant started = nwReply->property("started"); + if (started.isValid() && started.canConvert()) + { + const qint64 now = QDateTime::currentMSecsSinceEpoch(); + const qint64 start = started.value(); + headerResponse.setLoadTimeMs(now - start); + } + + const QDateTime lastModified = nwReply->header(QNetworkRequest::LastModifiedHeader).toDateTime(); + const qulonglong size = static_cast(nwReply->header(QNetworkRequest::ContentLengthHeader).toULongLong()); + headerResponse.setUpdateTimestamp(lastModified); + headerResponse.setContentLengthHeader(size); + + if (nwReply->error() == QNetworkReply::NoError) + { + // do not close because of obtaining data + return true; + } else { - // no valid response const QString error(nwReply->errorString()); const QString url(nwReply->url().toString()); nwReply->abort(); - datastoreResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, - QString("Reading data failed: " + error + " " + url))); + headerResponse.setMessage(CStatusMessage(this, CStatusMessage::SeverityError, + QString("Reading data failed: " + error + " " + url))); + return false; } - return datastoreResponse; } CDatabaseReader::JsonDatastoreResponse CDatabaseReader::setStatusAndTransformReplyIntoDatastoreResponse(QNetworkReply *nwReply) @@ -191,11 +225,50 @@ namespace BlackCore static const QDateTime e; const CDbInfoList il(infoList()); if (il.isEmpty() || entity == CEntityFlags::NoEntity) { return e; } - CDbInfo info = il.findFirstByEntityOrDefault(entity); + + // for some entities there can be more than one entry because of the + // raw tables (see DB view last updates) + const CDbInfo info = il.findFirstByEntityOrDefault(entity); if (!info.isValid()) { return e; } return info.getUtcTimestamp(); } + QDateTime CDatabaseReader::getSharedFileTimestamp(CEntityFlags::Entity entity) const + { + const int key = static_cast(entity); + if (m_sharedFileResponses.contains(key)) + { + return this->m_sharedFileResponses[key].getUpdateTimestamp(); + } + else + { + return QDateTime(); + } + } + + bool CDatabaseReader::requestHeadersOfSharedFiles(const CEntityFlags::Entity &entities) + { + CEntityFlags::Entity allEntities(entities); + CEntityFlags::Entity currentEntity = CEntityFlags::iterateDbEntities(allEntities); + CUrl urlDbData = CGlobalSetup::buildDbDataDirectory(getWorkingSharedUrl()); + + int c = 0; + while (currentEntity != CEntityFlags::NoEntity) + { + const QString fileName = CDbInfo::entityToSharedFileName(currentEntity); + Q_ASSERT_X(!fileName.isEmpty(), Q_FUNC_INFO, "No file name for entity"); + CUrl url = urlDbData; + url.appendPath(fileName); + + const QString entityString = CEntityFlags::flagToString(currentEntity); + CLogMessage(this).info("Triggered read of header for shared file of %1") << entityString; + const QNetworkReply *reply = sApp->headerFromNetwork(url, { this, &CDatabaseReader::receivedSharedFileHeader }); + if (reply) { c++; } + currentEntity = CEntityFlags::iterateDbEntities(allEntities); + } + return c > 0; + } + int CDatabaseReader::getCountFromInfoObjects(CEntityFlags::Entity entity) const { static const QDateTime e; @@ -240,6 +313,22 @@ namespace BlackCore return oldS != currentS; } + void CDatabaseReader::receivedSharedFileHeader(QNetworkReply *nwReplyPtr) + { + // wrap pointer, make sure any exit cleans up reply + // required to use delete later as object is created in a different thread + QScopedPointer nwReply(nwReplyPtr); + if (this->isAbandoned()) { return; } + + const HeaderResponse headerResponse = this->transformReplyIntoHeaderResponse(nwReply.data()); + const QString fileName = nwReply->url().fileName(); + const CEntityFlags::Entity entity = CEntityFlags::singleEntityByName(fileName); + const int key = static_cast(entity); + this->m_sharedFileResponses[key] = headerResponse; + + emit this->sharedFileHeaderRead(entity, fileName, !headerResponse.hasWarningOrAboveMessage()); + } + bool CDatabaseReader::hasReceivedOkReply() const { QReadLocker rl(&this->m_statusLock); @@ -324,14 +413,13 @@ namespace BlackCore return CNetworkUtils::canConnect(url); } - CDatabaseReader::JsonDatastoreResponse CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent) + void CDatabaseReader::stringToDatastoreResponse(const QString &jsonContent, JsonDatastoreResponse &datastoreResponse) { - CDatabaseReader::JsonDatastoreResponse datastoreResponse; if (jsonContent.isEmpty()) { datastoreResponse.setMessage(CStatusMessage(getLogCategories(), CStatusMessage::SeverityError, "Empty string, no data")); - datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); - return datastoreResponse; + datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc()); + return; } const QJsonDocument jsonResponse = QJsonDocument::fromJson(jsonContent.toUtf8()); @@ -339,17 +427,16 @@ namespace BlackCore { // directly an array, no further info datastoreResponse.setJsonArray(jsonResponse.array()); - datastoreResponse.m_updated = QDateTime::currentDateTimeUtc(); + datastoreResponse.setUpdateTimestamp(QDateTime::currentDateTimeUtc()); } else { const QJsonObject responseObject(jsonResponse.object()); datastoreResponse.setJsonArray(responseObject["data"].toArray()); const QString ts(responseObject["latest"].toString()); - datastoreResponse.m_updated = ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts); - datastoreResponse.m_restricted = responseObject["restricted"].toBool(); + datastoreResponse.setUpdateTimestamp(ts.isEmpty() ? QDateTime::currentDateTimeUtc() : CDatastoreUtility::parseTimestamp(ts)); + datastoreResponse.setRestricted(responseObject["restricted"].toBool()); } - return datastoreResponse; } void CDatabaseReader::JsonDatastoreResponse::setJsonArray(const QJsonArray &value) diff --git a/src/blackcore/db/databasereader.h b/src/blackcore/db/databasereader.h index 73475a14e..75cdf7e79 100644 --- a/src/blackcore/db/databasereader.h +++ b/src/blackcore/db/databasereader.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -44,21 +45,17 @@ namespace BlackCore Q_OBJECT public: - //! Response from our database - struct JsonDatastoreResponse + //! Header response part + struct HeaderResponse { - QJsonArray m_jsonArray; //!< JSON array data - QDateTime m_updated; //!< when was the latest updated? - int m_arraySize = -1; //!< size of array, if applicable (copied to member for debugging purposes) - bool m_restricted = false; //!< restricted reponse, only changed data - BlackMisc::CStatusMessage m_message; //!< last error or warning - - //! Any data? - bool isEmpty() const { return m_jsonArray.isEmpty(); } - - //! Number of elements - int size() const { return m_jsonArray.size(); } + private: + QDateTime m_updated; //!< when was the latest update? + BlackMisc::CStatusMessage m_message; //!< last error or warning + BlackMisc::Network::CUrl m_url; //!< loaded url + qulonglong m_contentLengthHeader = 0; //!< content length + qint64 m_loadTimeMs = -1; //!< how long did it take to load + public: //! Any timestamp? bool hasTimestamp() const { return m_updated.isValid(); } @@ -68,8 +65,17 @@ namespace BlackCore //! Is response newer? bool isNewer(qint64 mSecsSinceEpoch) const { return m_updated.toMSecsSinceEpoch() > mSecsSinceEpoch; } - //! Incremental data - bool isRestricted() const { return m_restricted; } + //! Get the update timestamp + const QDateTime &getUpdateTimestamp() const { return m_updated; } + + //! Set update timestamp, default normally "last-modified" + void setUpdateTimestamp(const QDateTime &updated) { m_updated = updated; } + + //! Header content length + qulonglong getContentLengthHeader() const { return m_contentLengthHeader; } + + //! Set the content length + void setContentLengthHeader(qulonglong size) { m_contentLengthHeader = size; } //! Error message? bool hasErrorMessage() const { return m_message.getSeverity() == BlackMisc::CStatusMessage::SeverityError; } @@ -83,6 +89,40 @@ namespace BlackCore //! Set the error/warning message void setMessage(const BlackMisc::CStatusMessage &lastErrorOrWarning) { m_message = lastErrorOrWarning; } + //! URL loaded + const BlackMisc::Network::CUrl &getUrl() const { return m_url; } + + //! Set the loaded URL + void setUrl(const BlackMisc::Network::CUrl &url) { m_url = url; } + + //! Load time in ms (from request to response) + qint64 getLoadTimeMs() const { return m_loadTimeMs; } + + //! Set the load time (delta start -> response received) + void setLoadTimeMs(qint64 deltaTime) { m_loadTimeMs = deltaTime; } + }; + + //! Response from our database (depneding on JSON DB backend generates) + struct JsonDatastoreResponse : public HeaderResponse + { + private: + QJsonArray m_jsonArray; //!< JSON array data + int m_arraySize = -1; //!< size of array, if applicable (copied to member for debugging purposes) + bool m_restricted = false; //!< restricted reponse, only changed data + + public: + //! Any data? + bool isEmpty() const { return m_jsonArray.isEmpty(); } + + //! Number of elements + int size() const { return m_jsonArray.size(); } + + //! Incremental data, restricted by query? + bool isRestricted() const { return m_restricted; } + + //! Incremental data, restricted by query + void setRestricted(bool restricted) { m_restricted = restricted; } + //! Get the JSON array QJsonArray getJsonArray() const { return m_jsonArray; } @@ -125,6 +165,12 @@ namespace BlackCore //! \sa BlackCore::Db::CInfoDataReader QDateTime getLatestEntityTimestampFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! Timestamp of shared file for entiry + QDateTime getSharedFileTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Request header of shared file + bool requestHeadersOfSharedFiles(const BlackMisc::Network::CEntityFlags::Entity &entities); + //! Count from info objects //! \sa BlackCore::Db::CInfoDataReader int getCountFromInfoObjects(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -141,18 +187,23 @@ namespace BlackCore //! Name of parameter for latest id static const QString ¶meterLatestId(); - //! sift DB server reachable? + //! swift DB server reachable? static bool canPingSwiftServer(); - //! Transform JSON data to response struct - static JsonDatastoreResponse stringToDatastoreResponse(const QString &jsonContent); + //! Transform JSON data to response struct data + //! \private used also for samples + static void stringToDatastoreResponse(const QString &jsonContent, CDatabaseReader::JsonDatastoreResponse &datastoreResponse); signals: //! Combined read signal void dataRead(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); + //! Header of shared file read + void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); + protected: CDatabaseReaderConfigList m_config; //!< DB reder configuration + QMap m_sharedFileResponses; //!< file responses of the shared files QString m_statusMessage; //!< Returned status message from watchdog QNetworkReply::NetworkError m_1stReplyStatus = QNetworkReply::UnknownServerError; //!< Successful connection? bool m_1stReplyReceived = false; //!< Successful connection? Does not mean data / authorizations are correct @@ -180,7 +231,7 @@ namespace BlackCore //! Obtain a working shared URL static BlackMisc::Network::CUrl getWorkingSharedUrl(); - //! \name Cache access + //! \name Cache access //! @{ //! Synchronize caches for given entities virtual void synchronizeCaches(BlackMisc::Network::CEntityFlags::Entity entities) = 0; @@ -202,9 +253,18 @@ namespace BlackCore //! @} private: + //! Received a reply of a header for a shared file + void receivedSharedFileHeader(QNetworkReply *nwReplyPtr); + //! Check if terminated or error, otherwise split into array of objects JsonDatastoreResponse transformReplyIntoDatastoreResponse(QNetworkReply *nwReply) const; + //! Check if terminated or error, otherwise set header information + HeaderResponse transformReplyIntoHeaderResponse(QNetworkReply *nwReply) const; + + //! Set the header part + bool setHeaderInfoPart(HeaderResponse &headerResponse, QNetworkReply *nwReply) const; + //! Feedback about connection status //! \threadsafe void setReplyStatus(QNetworkReply::NetworkError status, const QString &message = ""); diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index a54df1de8..93b151bd6 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -332,6 +332,27 @@ namespace BlackCore } } + QDateTime CWebDataServices::getSharedFileTimestamp(CEntityFlags::Entity entity) const + { + const CDatabaseReader *reader = this->getDbReader(entity); + if (reader) + { + return reader->getSharedFileTimestamp(entity); + } + else + { + return QDateTime(); + } + } + + bool CWebDataServices::requestHeaderOfSharedFile(CEntityFlags::Entity entity) + { + Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); + CDatabaseReader *reader = this->getDbReader(entity); + if (!reader) { return false; } + return reader->requestHeadersOfSharedFiles(entity); + } + int CWebDataServices::getCacheCount(CEntityFlags::Entity entity) const { Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); @@ -618,7 +639,7 @@ namespace BlackCore if (this->m_databaseWriter) { this->m_databaseWriter->gracefulShutdown(); } } - CEntityFlags::Entity CWebDataServices::allDbEntiiesUsed() const + CEntityFlags::Entity CWebDataServices::allDbEntiiesForUsedReaders() const { // obtain entities from real readers (means when reader is really used) CEntityFlags::Entity entities = CEntityFlags::NoEntity; @@ -671,9 +692,13 @@ namespace BlackCore { this->m_infoDataReader = new CInfoDataReader(this, dbReaderConfig); c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); - Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO info object signals"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed"); + + // relay signal c = connect(this->m_infoDataReader, &CInfoDataReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed info data"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Info object connect failed"); + + // start reading this->m_infoDataReader->start(QThread::LowPriority); QTimer::singleShot(0, [this]() { this->m_infoDataReader->read(CEntityFlags::InfoObjectEntity, QDateTime()); }); } @@ -711,7 +736,7 @@ namespace BlackCore if (flags.testFlag(CWebReaderFlags::WebReaderFlag::VatsimDataReader)) { this->m_vatsimDataFileReader = new CVatsimDataFileReader(this); - c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_dataFileRead); + c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataFileRead, this, &CWebDataServices::ps_vatsimDataFileRead); Q_ASSERT_X(c, Q_FUNC_INFO, "VATSIM data reader signals"); c = connect(this->m_vatsimDataFileReader, &CVatsimDataFileReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed VATSIM data file"); @@ -738,9 +763,11 @@ namespace BlackCore { this->m_icaoDataReader = new CIcaoDataReader(this, dbReaderConfig); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); - Q_ASSERT_X(c, Q_FUNC_INFO, "ICAO reader signals"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); c = connect(this->m_icaoDataReader, &CIcaoDataReader::dataRead, this, &CWebDataServices::dataRead); Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); + c = connect(this->m_icaoDataReader, &CIcaoDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect ICAO reader signals"); this->m_icaoDataReader->start(QThread::LowPriority); } @@ -749,9 +776,11 @@ namespace BlackCore { this->m_modelDataReader = new CModelDataReader(this, dbReaderConfig); c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); - Q_ASSERT_X(c, Q_FUNC_INFO, "Model reader signals"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_modelDataReader, &CModelDataReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed models"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); + c = connect(this->m_modelDataReader, &CModelDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); this->m_modelDataReader->start(QThread::LowPriority); } @@ -760,11 +789,17 @@ namespace BlackCore { this->m_airportDataReader = new CAirportDataReader(this, dbReaderConfig); c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::ps_readFromSwiftDb); - Q_ASSERT_X(c, Q_FUNC_INFO, "Airport reader signals"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); c = connect(this->m_airportDataReader, &CAirportDataReader::dataRead, this, &CWebDataServices::dataRead); - Q_ASSERT_X(c, Q_FUNC_INFO, "connect failed for airports"); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); + c = connect(this->m_airportDataReader, &CAirportDataReader::sharedFileHeaderRead, this, &CWebDataServices::sharedFileHeaderRead); + Q_ASSERT_X(c, Q_FUNC_INFO, "Cannot connect Model reader signals"); this->m_airportDataReader->start(QThread::LowPriority); } + + // Trigger Shared file headers loading + //! \todo refine, check if really needed to load the headers here + QTimer::singleShot(0, [this]() { this->triggerLoadingOfSharedFilesHeaders(); }); } CDatabaseReader *CWebDataServices::getDbReader(CEntityFlags::Entity entity) const @@ -772,7 +807,7 @@ namespace BlackCore Q_ASSERT_X(CEntityFlags::isSingleEntity(entity), Q_FUNC_INFO, "Need single entity"); Q_ASSERT_X(CEntityFlags::anySwiftDbEntity(entity), Q_FUNC_INFO, "No swift DB entity"); - CWebReaderFlags::WebReader wr = CWebReaderFlags::entityToReader(entity); + const CWebReaderFlags::WebReader wr = CWebReaderFlags::entityToReader(entity); switch (wr) { case CWebReaderFlags::IcaoDataReader: return this->m_icaoDataReader; @@ -801,7 +836,7 @@ namespace BlackCore CLogMessage(this).info("Read %1 METARs") << metars.size(); } - void CWebDataServices::ps_dataFileRead(int lines) + void CWebDataServices::ps_vatsimDataFileRead(int lines) { CLogMessage(this).info("Read VATSIM data file, %1 lines") << lines; } @@ -829,18 +864,13 @@ namespace BlackCore } this->m_swiftDbEntitiesRead |= entity; - const int allUsedEntities = static_cast(this->allDbEntiiesUsed()); + const int allUsedEntities = static_cast(this->allDbEntiiesForUsedReaders()); if (((static_cast(this->m_swiftDbEntitiesRead)) & allUsedEntities) == allUsedEntities) { emit allSwiftDbDataRead(); } } - void CWebDataServices::ps_setupChanged() - { - // void - } - void CWebDataServices::readDeferredInBackground(CEntityFlags::Entity entities, int delayMs) { if (entities == CEntityFlags::NoEntity) { return; } @@ -959,7 +989,7 @@ namespace BlackCore bool CWebDataServices::readDbDataFromDisk(const QString &dir, bool inBackground) { if (dir.isEmpty()) { return false; } - QDir directory(dir); + const QDir directory(dir); if (!directory.exists()) { return false; } bool s = false; @@ -977,4 +1007,37 @@ namespace BlackCore } return s; } + + bool CWebDataServices::triggerLoadingOfSharedFilesHeaders(CEntityFlags::Entity requestedEntities) + { + CEntityFlags::Entity triggeredEntities = CEntityFlags::NoEntity; + if (this->m_modelDataReader) + { + const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::ModelReader); + if (entities != CEntityFlags::NoEntity) + { + triggeredEntities |= entities; + this->m_modelDataReader->requestHeadersOfSharedFiles(entities); + } + } + if (this->m_icaoDataReader) + { + const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::IcaoDataReader); + if (entities != CEntityFlags::NoEntity) + { + triggeredEntities |= entities; + this->m_icaoDataReader->requestHeadersOfSharedFiles(entities); + } + } + if (this->m_airportDataReader) + { + const CEntityFlags::Entity entities = requestedEntities & CWebReaderFlags::allEntitiesForReaders(CWebReaderFlags::AirportReader); + if (entities != CEntityFlags::NoEntity) + { + triggeredEntities |= entities; + this->m_airportDataReader->requestHeadersOfSharedFiles(entities); + } + } + return triggeredEntities != CEntityFlags::NoEntity; + } } // ns diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 4d195073b..701c271cf 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -116,8 +116,8 @@ namespace BlackCore //! Reader flags CWebReaderFlags::WebReader getReaderFlags() const { return m_readers; } - //! All DB entities for those used and not ignored - BlackMisc::Network::CEntityFlags::Entity allDbEntiiesUsed() const; + //! All DB entities for those readers used and not ignored + BlackMisc::Network::CEntityFlags::Entity allDbEntiiesForUsedReaders() const; //! FSD servers //! \threadsafe @@ -317,6 +317,10 @@ namespace BlackCore //! Trigger reload from DB, only loads the DB data and bypasses the caches checks and info objects BlackMisc::Network::CEntityFlags::Entity triggerReloadFromDb(BlackMisc::Network::CEntityFlags::Entity whatToRead, const QDateTime &newerThan = QDateTime()); + //! Trigger loading of the HTTP headers for the shared files + //! \note allows to obtain the timestamps + bool triggerLoadingOfSharedFilesHeaders(BlackMisc::Network::CEntityFlags::Entity requestedEntities = BlackMisc::Network::CEntityFlags::AllDbEntities); + //! Corresponding cache timestamp if applicable //! \threadsafe QDateTime getCacheTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -325,6 +329,12 @@ namespace BlackCore //! \threadsafe QDateTime getDbLatestEntityTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! Corresponding shared file timestamp + QDateTime getSharedFileTimestamp(BlackMisc::Network::CEntityFlags::Entity entity) const; + + //! Request (updated) HTTP header for shared file of entity + bool requestHeaderOfSharedFile(BlackMisc::Network::CEntityFlags::Entity entity); + //! Cache count for entity //! \threadsafe int getCacheCount(BlackMisc::Network::CEntityFlags::Entity entity) const; @@ -358,6 +368,9 @@ namespace BlackCore //! All swift DB data have been read void allSwiftDbDataRead(); + //! Header of shared file read + void sharedFileHeaderRead(BlackMisc::Network::CEntityFlags::Entity entity, const QString &fileName, bool success); + public slots: //! Call CWebDataServices::readInBackground by single shot void readDeferredInBackground(BlackMisc::Network::CEntityFlags::Entity entities, int delayMs); @@ -373,14 +386,11 @@ namespace BlackCore void ps_receivedMetars(const BlackMisc::Weather::CMetarList &metars); //! Data file has been read - void ps_dataFileRead(int lines); + void ps_vatsimDataFileRead(int lines); //! Read finished from reader void ps_readFromSwiftDb(BlackMisc::Network::CEntityFlags::Entity entity, BlackMisc::Network::CEntityFlags::ReadState state, int number); - //! Setup changed - void ps_setupChanged(); - private: //! Init the readers void initReaders(CWebReaderFlags::WebReader flags);