diff --git a/src/blackcore/application.cpp b/src/blackcore/application.cpp index 8199b4b79..a7f66c990 100644 --- a/src/blackcore/application.cpp +++ b/src/blackcore/application.cpp @@ -616,20 +616,38 @@ namespace BlackCore QNetworkReply *CApplication::getFromNetwork(const CUrl &url, const CSlot &callback, int maxRedirects) { - return httpRequestImpl(url.toNetworkRequest(), callback, maxRedirects, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.get(request); }); + return getFromNetwork(url.toNetworkRequest(), NoLogRequestId, callback, maxRedirects); + } + + QNetworkReply *CApplication::getFromNetwork(const CUrl &url, int logId, const CSlot &callback, int maxRedirects) + { + return getFromNetwork(url.toNetworkRequest(), logId, callback, maxRedirects); } QNetworkReply *CApplication::getFromNetwork(const QNetworkRequest &request, const CSlot &callback, int maxRedirects) { - return httpRequestImpl(request, callback, maxRedirects, [ ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.get(request); }); + return getFromNetwork(request, NoLogRequestId, callback, maxRedirects); } - QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, const QByteArray &data, const CSlot &callback) + QNetworkReply *CApplication::getFromNetwork(const QNetworkRequest &request, int logId, const CSlot &callback, int maxRedirects) { - return httpRequestImpl(request, callback, -1, [ data ](QNetworkAccessManager & nam, const QNetworkRequest & request) { return nam.post(request, data); }); + return httpRequestImpl(request, logId, callback, maxRedirects, [](QNetworkAccessManager & qam, const QNetworkRequest & request) + { + QNetworkReply *nr = qam.get(request); + return nr; + }); } - QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, QHttpMultiPart *multiPart, const CSlot &callback) + QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, int logId, const QByteArray &data, const CSlot &callback) + { + return httpRequestImpl(request, logId, callback, NoRedirects, [ data ](QNetworkAccessManager & qam, const QNetworkRequest & request) + { + QNetworkReply *nr = qam.post(request, data); + return nr; + }); + } + + QNetworkReply *CApplication::postToNetwork(const QNetworkRequest &request, int logId, QHttpMultiPart *multiPart, const CSlot &callback) { if (!this->isNetworkAccessible()) { return nullptr; } if (QThread::currentThread() != this->m_accessManager->thread()) @@ -637,23 +655,22 @@ namespace BlackCore multiPart->moveToThread(this->m_accessManager->thread()); } - return httpRequestImpl(request, callback, -1, [ this, multiPart ](QNetworkAccessManager & qam, const QNetworkRequest & request) + return httpRequestImpl(request, logId, callback, NoRedirects, [ this, multiPart ](QNetworkAccessManager & qam, const QNetworkRequest & request) { - QNetworkReply *reply = qam.post(request, multiPart); - Q_ASSERT(reply); - multiPart->setParent(reply); - return reply; + QNetworkReply *nr = qam.post(request, multiPart); + multiPart->setParent(nr); + return nr; }); } QNetworkReply *CApplication::headerFromNetwork(const CUrl &url, const CSlot &callback, int maxRedirects) { - return httpRequestImpl(url.toNetworkRequest(), callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); }); + return headerFromNetwork(url.toNetworkRequest(), callback, maxRedirects); } QNetworkReply *CApplication::headerFromNetwork(const QNetworkRequest &request, const CSlot &callback, int maxRedirects) { - return httpRequestImpl(request, callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); }); + return httpRequestImpl(request, NoLogRequestId, callback, maxRedirects, [ ](QNetworkAccessManager & qam, const QNetworkRequest & request) { return qam.head(request); }); } QNetworkReply *CApplication::downloadFromNetwork(const CUrl &url, const QString &saveAsFileName, const BlackMisc::CSlot &callback, int maxRedirects) @@ -1310,7 +1327,7 @@ namespace BlackCore if (!QFileInfo::exists(handler)) { - return CStatusMessage(this).warning("%1 not found. Cannot init crash handler!") << handler; + return CStatusMessage(this).warning("Crashpad handler '%1' not found. Cannot init handler!") << handler; } const CUrl serverUrl = this->getGlobalSetup().getCrashReportServerUrl(); @@ -1342,7 +1359,9 @@ namespace BlackCore #endif } - QNetworkReply *CApplication::httpRequestImpl(const QNetworkRequest &request, const BlackMisc::CSlot &callback, int maxRedirects, std::function requestOrPostMethod) + QNetworkReply *CApplication::httpRequestImpl( + const QNetworkRequest &request, int logId, + const BlackMisc::CSlot &callback, int maxRedirects, std::function requestOrPostMethod) { if (this->isShuttingDown()) { return nullptr; } if (!this->isNetworkAccessible()) { return nullptr; } @@ -1351,7 +1370,7 @@ namespace BlackCore if (QThread::currentThread() != this->m_accessManager->thread()) { // run in QAM thread - QTimer::singleShot(0, this->m_accessManager, std::bind(&CApplication::httpRequestImpl, this, request, callback, maxRedirects, requestOrPostMethod)); + QTimer::singleShot(0, this->m_accessManager, std::bind(&CApplication::httpRequestImpl, this, request, logId, callback, maxRedirects, requestOrPostMethod)); return nullptr; // not yet started } @@ -1360,11 +1379,12 @@ namespace BlackCore CNetworkUtils::ignoreSslVerification(copiedRequest); CNetworkUtils::setSwiftUserAgent(copiedRequest); - // If URL is one of the shared urls, add swift client SSL certificate + // If URL is one of the shared URLs, add swift client SSL certificate CNetworkUtils::setSwiftClientSslCertificate(copiedRequest, getGlobalSetup().getSwiftSharedUrls()); QNetworkReply *reply = requestOrPostMethod(*this->m_accessManager, copiedRequest); reply->setProperty("started", QVariant(QDateTime::currentMSecsSinceEpoch())); + reply->setProperty(CUrlLog::propertyNameId(), QVariant(logId)); if (callback) { Q_ASSERT_X(callback.object(), Q_FUNC_INFO, "Need callback object (to determine thread)"); @@ -1381,7 +1401,7 @@ namespace BlackCore { QNetworkRequest redirectRequest(redirectUrl); const int redirectsLeft = maxRedirects - 1; - QTimer::singleShot(0, this, std::bind(&CApplication::httpRequestImpl, this, redirectRequest, callback, redirectsLeft, requestOrPostMethod)); + QTimer::singleShot(0, this, std::bind(&CApplication::httpRequestImpl, this, redirectRequest, logId, callback, redirectsLeft, requestOrPostMethod)); return; } } diff --git a/src/blackcore/application.h b/src/blackcore/application.h index ca1c9744c..9edbaa4fd 100644 --- a/src/blackcore/application.h +++ b/src/blackcore/application.h @@ -365,42 +365,56 @@ namespace BlackCore //! Wait for setup data by calling the event loop and waiting until everything is ready BlackMisc::CStatusMessageList waitForSetup(); + public: + static constexpr int NoRedirects = -1; //!< network request not allowing redirects + static constexpr int NoLogRequestId = -1; //!< network request without logging + static constexpr int DefaultMaxRedirects = 2; //!< network request, default for max.redirects + //! Request to get network reply //! \threadsafe QNetworkReply *getFromNetwork(const BlackMisc::Network::CUrl &url, - const BlackMisc::CSlot &callback, int maxRedirects = 2); + const BlackMisc::CSlot &callback, int maxRedirects = DefaultMaxRedirects); + + //! Request to get network reply, supporting BlackMisc::Network::CUrlLog + //! \threadsafe + QNetworkReply *getFromNetwork(const BlackMisc::Network::CUrl &url, int logId, + const BlackMisc::CSlot &callback, int maxRedirects = DefaultMaxRedirects); //! Request to get network reply //! \threadsafe QNetworkReply *getFromNetwork(const QNetworkRequest &request, - const BlackMisc::CSlot &callback, int maxRedirects = 2); + const BlackMisc::CSlot &callback, int maxRedirects = DefaultMaxRedirects); + + //! Request to get network reply, supporting BlackMisc::Network::CUrlLog + //! \threadsafe + QNetworkReply *getFromNetwork(const QNetworkRequest &request, int logId, + const BlackMisc::CSlot &callback, int maxRedirects = DefaultMaxRedirects); //! Post to network //! \threadsafe - QNetworkReply *postToNetwork(const QNetworkRequest &request, const QByteArray &data, + QNetworkReply *postToNetwork(const QNetworkRequest &request, int logId, const QByteArray &data, const BlackMisc::CSlot &callback); //! Post to network //! \note This method takes ownership over \c multiPart. //! \threadsafe - QNetworkReply *postToNetwork(const QNetworkRequest &request, QHttpMultiPart *multiPart, + QNetworkReply *postToNetwork(const QNetworkRequest &request, int logId, QHttpMultiPart *multiPart, const BlackMisc::CSlot &callback); //! Request to get network repy using HTTP's HEADER method //! \threadsafe QNetworkReply *headerFromNetwork(const BlackMisc::Network::CUrl &url, - const BlackMisc::CSlot &callback, int maxRedirects = -1); + const BlackMisc::CSlot &callback, int maxRedirects = NoRedirects); //! Request to get network repy using HTTP's HEADER method //! \threadsafe QNetworkReply *headerFromNetwork(const QNetworkRequest &request, - const BlackMisc::CSlot &callback, int maxRedirects = -1); + const BlackMisc::CSlot &callback, int maxRedirects = NoRedirects); - public: // downloadFromNetwork no slot to avoid issue with CSlot, see T125 //! Download file from network and store it as passed //! \threadsafe QNetworkReply *downloadFromNetwork(const BlackMisc::Network::CUrl &url, const QString &saveAsFileName, - const BlackMisc::CSlot &callback, int maxRedirects = 2); + const BlackMisc::CSlot &callback, int maxRedirects = DefaultMaxRedirects); signals: //! Setup available (cache, web load, ..) or failed to load setup @@ -501,7 +515,9 @@ namespace BlackCore BlackMisc::CStatusMessageList asyncWebAndContextStart(); //! Implementation for getFromNetwork(), postToNetwork() and headerFromNetwork() + //! \return QNetworkReply reply will only be returned, if the QNetworkAccessManager is in the same thread QNetworkReply *httpRequestImpl(const QNetworkRequest &request, + int logId, const BlackMisc::CSlot &callback, int maxRedirects, std::function requestOrPostMethod); diff --git a/src/blackcore/webdataservices.cpp b/src/blackcore/webdataservices.cpp index 443bc6e7e..0554144ab 100644 --- a/src/blackcore/webdataservices.cpp +++ b/src/blackcore/webdataservices.cpp @@ -412,6 +412,30 @@ namespace BlackCore return this->getInfoObjectCount(entity, m_sharedInfoDataReader); } + QString CWebDataServices::getDbReadersLog(const QString separator) const + { + QStringList report; + if (m_dbInfoDataReader) { report << m_dbInfoDataReader->getName() + ": " + m_dbInfoDataReader->getReadLog().getSummary(); } + if (m_sharedInfoDataReader) { report << m_sharedInfoDataReader->getName() + ": " + m_sharedInfoDataReader->getReadLog().getSummary(); } + + if (m_airportDataReader) { report << m_airportDataReader->getName() + ": " + m_airportDataReader->getReadLog().getSummary(); } + if (m_icaoDataReader) { report << m_icaoDataReader->getName() + ": " + m_icaoDataReader->getReadLog().getSummary(); } + if (m_modelDataReader) { report << m_modelDataReader->getName() + ": " + m_modelDataReader->getReadLog().getSummary(); } + if (m_databaseWriter) { report << m_databaseWriter->getName() + ": " + m_databaseWriter->getWriteLog().getSummary(); } + return report.join(separator); + } + + QString CWebDataServices::getReadersLog(const QString separator) const + { + const QString db = this->getDbReadersLog(separator); + QStringList report; + if (m_vatsimBookingReader) { report << m_vatsimBookingReader->getName() + ": " + m_vatsimBookingReader->getReadLog().getSummary(); } + if (m_vatsimMetarReader) { report << m_vatsimMetarReader->getName() + ": " + m_vatsimMetarReader->getReadLog().getSummary(); } + if (m_vatsimStatusReader) { report << m_vatsimStatusReader->getName() + ": " + m_vatsimStatusReader->getReadLog().getSummary(); } + if (report.isEmpty()) { return db; } + return report.join(separator) + separator + db; + } + CDistributorList CWebDataServices::getDistributors() const { if (m_modelDataReader) { return m_modelDataReader->getDistributors(); } @@ -1165,7 +1189,7 @@ namespace BlackCore if (!this->waitForDbInfoObjectsThenRead(entities)) { return; } } - const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFile(entities); + const bool waitForSharedInfoFile = m_dbReaderConfig.needsSharedInfoFile(entities) && !m_sharedInfoDataReader->areAllInfoObjectsRead(); if (waitForSharedInfoFile) { // do not read yet, will call this function again after some time @@ -1181,67 +1205,68 @@ namespace BlackCore bool CWebDataServices::waitForDbInfoObjectsThenRead(CEntityFlags::Entity entities) { Q_ASSERT_X(m_dbInfoDataReader, Q_FUNC_INFO, "need reader"); + if (m_dbInfoDataReader->areAllInfoObjectsRead()) { return true; } if (!m_dbInfoObjectTimeout.isValid()) { m_dbInfoObjectTimeout = QDateTime::currentDateTimeUtc().addMSecs(10 * 1000); } const bool read = this->waitForInfoObjectsThenRead(entities, "DB", m_dbInfoDataReader, m_dbInfoObjectTimeout); - if (read) { m_dbInfoObjectTimeout = QDateTime(); } // reset to null return read; } bool CWebDataServices::waitForSharedInfoObjectsThenRead(CEntityFlags::Entity entities) { Q_ASSERT_X(m_sharedInfoDataReader, Q_FUNC_INFO, "need reader"); + if (m_sharedInfoDataReader->areAllInfoObjectsRead()) { return true; } if (!m_sharedInfoObjectsTimeout.isValid()) { m_sharedInfoObjectsTimeout = QDateTime::currentDateTimeUtc().addMSecs(10 * 1000); } const bool read = this->waitForInfoObjectsThenRead(entities, "shared", m_sharedInfoDataReader, m_sharedInfoObjectsTimeout); - if (read) { m_sharedInfoObjectsTimeout = QDateTime(); } // reset to null return read; } - bool CWebDataServices::waitForInfoObjectsThenRead(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *reader, const QDateTime &timeOut) + bool CWebDataServices::waitForInfoObjectsThenRead(CEntityFlags::Entity entities, const QString &info, CInfoDataReader *infoReader, QDateTime &timeOut) { - Q_ASSERT_X(reader, Q_FUNC_INFO, "Need info data reader"); + Q_ASSERT_X(infoReader, Q_FUNC_INFO, "Need info data reader"); // this will called for each entity readers, i.e. model reader, ICAO reader ... const int waitForInfoObjectsMs = 1000; // ms + if (infoReader->areAllInfoObjectsRead()) + { + // we have all data and carry on + CLogMessage(this).info("Info objects (%1) triggered for '%2' loaded from '%3'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString(); + timeOut = QDateTime(); // reset to null + return true; // no need to wait any longer + } + // try to read if not timed out - if (QDateTime::currentDateTimeUtc() > timeOut) + if (timeOut.isValid() && QDateTime::currentDateTimeUtc() > timeOut) { const QString timeOutString = timeOut.toString(); CLogMessage(this).warning("Could not read '%1' info objects for '%2' from '%3', time out '%4'. Marking reader '%5' as failed and continue.") << info << CEntityFlags::flagToString(entities) - << reader->getInfoObjectsUrl().toQString() << timeOutString - << reader->getName(); + << infoReader->getInfoObjectsUrl().toQString() << timeOutString + << infoReader->getName(); // continue here and read data without info objects - reader->setMarkedAsFailed(true); + infoReader->setMarkedAsFailed(true); + // no timeout reset here return true; // carry on, regardless of situation } - else if (reader->hasReceivedFirstReply()) + + if (infoReader->hasReceivedFirstReply()) { - if (reader->areAllInfoObjectsRead()) + // we have received a response, but not all data yet + if (infoReader->hasReceivedOkReply()) { - // we have all data and carry on - CLogMessage(this).info("Info objects (%1) for '%2' loaded from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString(); - return true; // no need to wait any longer + // ok, this means we are parsing + this->readDeferredInBackground(entities, waitForInfoObjectsMs); + CLogMessage(this).info("Waiting for objects (%1) for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString(); + return false; // wait } else { - // we have received a response, but not all data yet - if (reader->hasReceivedOkReply()) - { - // ok, this means we are parsing - this->readDeferredInBackground(entities, waitForInfoObjectsMs); - CLogMessage(this).info("Waiting for objects (%1) for '%2' from '%3'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString(); - return false; // wait - } - else - { - // we have a response, but a failure, means server is alive, but responded with error - // such an error (access, ...) normally will not go away - CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << reader->getInfoObjectsUrl().toQString() << reader->getStatusMessage(); - reader->setMarkedAsFailed(true); - return true; // carry on, regardless of situation - } + // we have a response, but a failure, means server is alive, but responded with error + // such an error (access, ...) normally will not go away + CLogMessage(this).error("Info objects (%1) loading for '%2' failed from '%3', '%4'") << info << CEntityFlags::flagToString(entities) << infoReader->getInfoObjectsUrl().toQString() << infoReader->getStatusMessage(); + infoReader->setMarkedAsFailed(true); + return true; // carry on, regardless of situation } } else diff --git a/src/blackcore/webdataservices.h b/src/blackcore/webdataservices.h index 9a846f672..dcbe5f26a 100644 --- a/src/blackcore/webdataservices.h +++ b/src/blackcore/webdataservices.h @@ -395,6 +395,12 @@ namespace BlackCore //! \threadsafe int getSharedInfoObjectCount(BlackMisc::Network::CEntityFlags::Entity entity) const; + //! For all available DB readers the log info is generated + QString getDbReadersLog(const QString separator = "\n") const; + + //! For all available readers the log info is generated + QString getReadersLog(const QString separator = "\n") const; + //! Has already connect swift DB? bool hasConnectedSwiftDb() const; @@ -513,7 +519,7 @@ namespace BlackCore //! Wait for info objects to be read //! \return true means info objects available - bool waitForInfoObjectsThenRead(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *reader, const QDateTime &timeOut); + bool waitForInfoObjectsThenRead(BlackMisc::Network::CEntityFlags::Entity entities, const QString &info, BlackCore::Db::CInfoDataReader *infoReader, QDateTime &timeOut); CWebReaderFlags::WebReader m_readers = CWebReaderFlags::WebReaderFlag::None; //!< which readers are available BlackMisc::Network::CEntityFlags::Entity m_entitiesPeriodicallyRead = BlackMisc::Network::CEntityFlags::NoEntity; //!< entities permanently updated by timers diff --git a/src/blackgui/guiapplication.cpp b/src/blackgui/guiapplication.cpp index c84f95cae..be5b9638a 100644 --- a/src/blackgui/guiapplication.cpp +++ b/src/blackgui/guiapplication.cpp @@ -10,6 +10,7 @@ #include "blackconfig/buildconfig.h" #include "blackcore/context/contextnetwork.h" #include "blackcore/data/globalsetup.h" +#include "blackcore/webdataservices.h" #include "blackgui/components/applicationclosedialog.h" #include "blackgui/components/downloadandinstalldialog.h" #include "blackgui/components/aboutdialog.h" @@ -56,6 +57,7 @@ using namespace BlackMisc; using namespace BlackMisc::Db; using namespace BlackMisc::Network; using namespace BlackGui::Components; +using namespace BlackCore; using namespace BlackCore::Data; BlackGui::CGuiApplication *sGui = nullptr; // set by constructor @@ -457,6 +459,16 @@ namespace BlackGui }); Q_ASSERT_X(c, Q_FUNC_INFO, "Connect failed"); + if (this->hasWebDataServices()) + { + a = menu.addAction("Services log"); + c = connect(a, &QAction::triggered, this, [a, this]() + { + this->displayTextInConsole(this->getWebDataServices()->getReadersLog()); + }); + Q_ASSERT_X(c, Q_FUNC_INFO, "Connect failed"); + } + a = menu.addAction("Metadata (slow)"); c = connect(a, &QAction::triggered, this, [a, this]() {